使用SPIN技术对LLM进行自我博弈微调训练

# Import necessary libraries
 from datasets import load_dataset
 import pandas as pd
 
 # Load the Ultrachat 200k dataset
 ultrachat_dataset = load_dataset("HuggingFaceH4/ultrachat_200k")
 
 # Initialize an empty DataFrame
 combined_df = pd.DataFrame()
 
 # Loop through all the keys in the Ultrachat dataset
 for key in ultrachat_dataset.keys():
    # Convert each dataset key to a pandas DataFrame and concatenate it with the existing DataFrame
    combined_df = pd.concat([combined_df, pd.DataFrame(ultrachat_dataset[key])])
 
 # Shuffle the combined DataFrame and reset the index
 combined_df = combined_df.sample(frac=1, random_state=123).reset_index(drop=True)
 
 # Select the first 50,000 rows from the shuffled DataFrame
 ultrachat_50k_sample = combined_df.head(50000)

# for storing each template in a list
 templates_data = []
 
 for index, row in ultrachat_50k_sample.iterrows():
    messages = row['messages']
     
    # Check if there are at least two messages (user and assistant)
    if len(messages) >= 2:
        user_message = messages[0]['content']
        assistant_message = messages[1]['content']
         
        # Create the template
        instruction_response_template = f"### Instruction: {user_message}\n\n### Response: {assistant_message}"
         
        # Append the template to the list
        templates_data.append({'Template': instruction_response_template})
 
 # Create a new DataFrame with the generated templates (ground truth)
 ground_truth_df = pd.DataFrame(templates_data)

 # Importing the PyTorch library
 import torch
 
 # Importing the neural network module from PyTorch
 import torch.nn as nn
 
 # Importing the DeepSpeed library for distributed training
 import deepspeed
 
 # Importing the AutoTokenizer and AutoModelForCausalLM classes from the transformers library
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
 # Loading the zephyr-7b-sft-full model from HuggingFace
 tokenizer = AutoTokenizer.from_pretrained("alignment-handbook/zephyr-7b-sft-full")
 model = AutoModelForCausalLM.from_pretrained("alignment-handbook/zephyr-7b-sft-full")
 
 # Initializing DeepSpeed Zero with specific configuration settings
 deepspeed_config = deepspeed.config.Config(train_batch_size=64, train_micro_batch_size_per_gpu=4)
 model, optimizer, _, _ = deepspeed.initialize(model=model, config=deepspeed_config, model_parameters=model.parameters())
 
 # Defining the optimizer and setting the learning rate using RMSprop
 optimizer = deepspeed.optim.RMSprop(optimizer, lr=5e-7)
 
 # Setting up a learning rate scheduler using LambdaLR from PyTorch
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.2 ** epoch)
 
 # Setting hyperparameters for training
 num_epochs = 3
 max_seq_length = 2048
 beta = 0.1

# zephyr-sft-dataframe (that contains output that will be improved while training)
 zephyr_sft_output = pd.DataFrame(columns=['prompt', 'generated_output'])
 
 # Looping through each row in the 'ultrachat_50k_sample' dataframe
 for index, row in ultrachat_50k_sample.iterrows():
    # Extracting the 'prompt' column value from the current row
    prompt = row['prompt']
     
    # Generating output for the current prompt using the Zephyr model
    input_ids = tokenizer(prompt, return_tensors="pt").input_ids
    output = model.generate(input_ids, max_length=200, num_beams=5, no_repeat_ngram_size=2, top_k=50, top_p=0.95)
     
    # Decoding the generated output to human-readable text
    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
     
    # Appending the current prompt and its generated output to the new dataframe 'zephyr_sft_output'
    zephyr_sft_output = zephyr_sft_output.append({'prompt': prompt, 'generated_output': generated_text}, ignore_index=True)

 # Conditional Probability Function of input text
 def compute_conditional_probability(tokenizer, model, input_text):
    # Tokenize the input text and convert it to PyTorch tensors
    inputs = tokenizer([input_text], return_tensors="pt")
 
    # Generate text using the model, specifying additional parameters
    outputs = model.generate(**inputs, return_dict_in_generate=True, output_scores=True)
 
    # Assuming 'transition_scores' is the logits for the generated tokens
    transition_scores = model.compute_transition_scores(outputs.sequences, outputs.scores, normalize_logits=True)
 
    # Get the length of the input sequence
    input_length = inputs.input_ids.shape[1]
 
    # Assuming 'transition_scores' is the logits for the generated tokens
    logits = torch.tensor(transition_scores)
 
    # Apply softmax to obtain probabilities
    probs = torch.nn.functional.softmax(logits, dim=-1)
 
    # Extract the generated tokens from the output
    generated_tokens = outputs.sequences[:, input_length:]
 
    # Compute conditional probability
    conditional_probability = 1.0
    for prob in probs[0]:
        token_probability = prob.item()
        conditional_probability *= token_probability
 
    return conditional_probability

 def LSPIN_loss(model, updated_model, tokenizer, input_text, lambda_val=0.01):
    # Initialize conditional probability using the original model and input text
    cp = compute_conditional_probability(tokenizer, model, input_text)
 
    # Update conditional probability using the updated model and input text
    cp_updated = compute_conditional_probability(tokenizer, updated_model, input_text)
 
    # Calculate conditional probabilities for ground truth data
    p_theta_ground_truth = cp(tokenizer, model, input_text)
    p_theta_t_ground_truth = cp(tokenizer, model, input_text)
 
    # Calculate conditional probabilities for synthetic data
    p_theta_synthetic = cp_updated(tokenizer, updated_model, input_text)
    p_theta_t_synthetic = cp_updated(tokenizer, updated_model, input_text)
 
    # Calculate likelihood ratios
    lr_ground_truth = p_theta_ground_truth / p_theta_t_ground_truth
    lr_synthetic = p_theta_synthetic / p_theta_t_synthetic
 
    # Compute the LSPIN loss
    loss = lambda_val * torch.log(lr_ground_truth) - lambda_val * torch.log(lr_synthetic)
 
    return loss

# Training loop
 for epoch in range(num_epochs):
     
    # Model with initial parameters
    initial_model = AutoModelForCausalLM.from_pretrained("alignment-handbook/zephyr-7b-sft-full")
   
    # Update the learning rate
    scheduler.step()
 
    # Initialize total loss for the epoch
    total_loss = 0.0
 
    # Generating Synthetic Data (Inner loop)
    for index, row in ultrachat_50k_sample.iterrows():
 
        # Rest of the code       
        ...
 
        # Output == prompt response dataframe
        zephyr_sft_output
 
    # Computing loss using LSPIN function
    for (index1, row1), (index2, row2) in zip(ultrachat_50k_sample.iterrows(), zephyr_sft_output.iterrows()):
        # Assuming 'prompt' and 'generated_output' are the relevant columns in zephyr_sft_output
        prompt = row1['prompt']
        generated_output = row2['generated_output']
 
        # Compute LSPIN loss
        updated_model = model # It will be replacing with updated model
        loss = LSPIN_loss(initial_model, updated_model, tokenizer, prompt)
 
        # Accumulate the loss
        total_loss += loss.item()
 
    # Backward pass
    loss.backward()
 
    # Update the parameters
    optimizer.step()
 
    # Update the value of beta
    if epoch == 2:
        beta = 5.0