1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model_path = 'Meta-Llama-3.1-8B-Instruct'
tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
model = AutoModelForCausalLM.from_pretrained(model_path).to("cuda")
optimizer = torch.optim.AdamW(model.parameters())

# 构造对话数据
dialog = [{"role": "system", "content": "You are a helpful assistant."},
          {"role": "user", "content": "天空为什么是蓝色的？"},
          {"role": "assistant", "content": "这是由于光的散射引起的。"}]

# 使用Chat Template将对话转换为模型输入格式
input = tokenizer.apply_chat_template(dialog, return_tensors="pt")
input = {k: v.to("cuda") for k, v in input.items()}

# 关键：设置labels和input_ids一致——模型自回归预测下一个token
input["labels"] = input["input_ids"].clone()

output = model(**input)
loss = output.loss  # 交叉熵损失
loss.backward()
optimizer.step()
optimizer.zero_grad()

model.save_pretrained("output_dir")

1
2
3
4

peft_config = LoraConfig(
    r=16,  # LoRA的秩
    ...
)

1
2
3
4
5

peft_config = LoraConfig(
    r=16,
    lora_alpha=16,  # 缩放因子
    ...
)

1
2
3
4
5

peft_config = LoraConfig(
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    ...
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from peft import LoraConfig, get_peft_model, TaskType

peft_config = LoraConfig(
    r=8,
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj",
                    "gate_proj", "down_proj", "up_proj"],
    task_type=TaskType.CAUSAL_LM,
    lora_alpha=16,
    lora_dropout=0.05
)
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()
# 输出类似：trainable params: 41,943,040 || all params: 8,000,000,000 || trainable%: 0.52%

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

from transformers import BitsAndBytesConfig
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,              # 启用4-bit量化
    bnb_4bit_use_double_quant=True, # 双重量化：对量化常数再做一次量化，进一步节约显存
    bnb_4bit_quant_type="nf4",      # 使用NormalFloat4量化类型，对正态分布权重更优
    bnb_4bit_compute_dtype=torch.bfloat16  # 计算时反量化到bfloat16精度
)

model = AutoModelForCausalLM.from_pretrained(
    model_path,
    quantization_config=bnb_config,
    torch_dtype=torch.float16
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

bnb_config = BitsAndBytesConfig(
    load_in_8bit=True,  # 启用8-bit量化
)

model = AutoModelForCausalLM.from_pretrained(
    model_id_or_path,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True
)

1
2
3

from peft import prepare_model_for_kbit_training

model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

neftune_noise_alpha = 10  # 噪声强度超参数

for inputs, loss_mask in data_loader:
    input_ids = inputs.pop("input_ids")
    # 获取嵌入层输出
    input_embeddings = model.base_model.model.model.embed_tokens(input_ids)

    # 计算噪声幅度：alpha / sqrt(L * d)
    # L = 序列长度, d = 嵌入维度
    dims = torch.tensor(input_embeddings.size(1) * input_embeddings.size(2))
    mag_norm = neftune_noise_alpha / torch.sqrt(dims)

    # 添加均匀分布噪声 U(-mag_norm, mag_norm)
    input_embeddings = input_embeddings + torch.zeros_like(input_embeddings).uniform_(
        -mag_norm, mag_norm
    )

    inputs["inputs_embeds"] = input_embeddings  # 用带噪声的嵌入替换原始input_ids
    # ... 继续正常的前向传播和反向传播

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

from trl import SFTTrainer, SFTConfig

sft_config = SFTConfig(
    output_dir="/tmp",
    neftune_noise_alpha=10,  # 只需设置这个参数即可启用NEFT
    per_device_train_batch_size=1,
    max_seq_length=100,
    num_train_epochs=10,
    logging_steps=10,
    logging_strategy="steps"
)

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=sft_config,
    data_collator=collator
)
trainer.train()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

from trl import DataCollatorForCompletionOnlyLM

# 定义response部分的起始标记——这个模板要与tokenizer的chat_template匹配
response_template = "<|start_header_id|>assistant<|end_header_id|>\n\n"
collator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=tokenizer)

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=sft_config,
    data_collator=collator  # 传入collator
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def sft_collate(batch, tokenizer, end_str, max_length):
    inputs = tokenizer(batch, max_length=max_length, padding=True, truncation=True)
    input_ids = inputs["input_ids"]
    input_len = len(input_ids[0])

    # 将response起始标记编码为token id序列
    end_ids = tokenizer(end_str)["input_ids"]
    end_id_len = len(end_ids)

    loss_mask = []
    for input_id in input_ids:
        # 从后向前搜索response起始标记的位置
        for i in range(len(input_id) - end_id_len, -1, -1):
            if input_id[i:i + end_id_len] == end_ids:
                # 构建mask：response部分为1，其余为0
                mask = [1] * (input_len - 1)
                mask[:i + end_id_len - 1] = [0] * (i + end_id_len - 1)
                loss_mask.append(mask)
                break
            if i == 0:  # 如果没找到（回答被截断），全部mask为0
                loss_mask.append([0] * (input_len - 1))

    inputs = {k: torch.tensor(v) for k, v in inputs.items()}
    loss_mask = torch.tensor(loss_mask)
    return inputs, loss_mask

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

for inputs, loss_mask in data_loader:
    inputs = {k: v.to("cuda") for k, v in inputs.items()}
    loss_mask = loss_mask.to("cuda")

    logits = model(**inputs).logits[:, :-1, :]   # 去掉最后一个位置
    labels = inputs["input_ids"][:, 1:]           # 去掉第一个位置（shift right）

    logits = logits.reshape(-1, logits.size(-1))
    labels = labels.reshape(-1)
    loss_mask = loss_mask.reshape(-1)

    # 先计算每个token的loss（不做reduction）
    loss = torch.nn.functional.cross_entropy(logits, labels, reduction="none")
    # 应用mask：只保留response部分的loss
    loss = loss * loss_mask
    # 对有效token求平均
    loss = loss.sum() / loss_mask.sum()

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

dialog = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "天空为什么是蓝色的？"},
    {"role": "assistant", "content": "这是由于光的散射引起的。"}
]

# tokenize=False 返回格式化后的文本字符串
chat_text = tokenizer.apply_chat_template(dialog, tokenize=False)

# tokenize=True（默认）返回token id
chat_tokens = tokenizer.apply_chat_template(dialog, return_tensors="pt")

1
2
3
4
5
6
7

<|begin_of_text|><|start_header_id|>system<|end_header_id|>

You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>

天空为什么是蓝色的？<|eot_id|><|start_header_id|>assistant<|end_header_id|>

这是由于光的散射引起的。<|eot_id|>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

train_prompt_style = """Below is an instruction that describes a task...

### Instruction:
You are a medical expert with advanced knowledge in clinical reasoning...

### Question:
{}

### Response:
<think>
{}
</think>
{}"""

EOS_TOKEN = tokenizer.eos_token  # 必须添加结束标记！

def formatting_prompts_func(examples):
    inputs = examples["Question"]
    cots = examples["Complex_CoT"]
    outputs = examples["Response"]
    texts = []
    for input, cot, output in zip(inputs, cots, outputs):
        text = train_prompt_style.format(input, cot, output) + EOS_TOKEN
        texts.append(text)
    return {"text": texts}

dataset = dataset.map(formatting_prompts_func, batched=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

import json

json_data = []
with open('DISC-Law-SFT-Triplet-released.jsonl', 'r', encoding='utf-8') as file:
    for line in file:
        data = json.loads(line)
        json_data.append(data)

template = []
for idx, data in enumerate(json_data):
    conversation = [
        {"from": "human", "value": data["input"]},
        {"from": "gpt", "value": data["output"]}
    ]
    template.append({
        "conversations": conversation,
        "system": "",
        "tools": ""
    })

# 保存为LLaMA-Factory所需的JSON格式
with open('train_data_law.json', 'w', encoding='utf-8') as f:
    json.dump(template, f, ensure_ascii=False, indent=2)

1
2
3

model_id_or_path = "DeepSeek-R1-Distill-Qwen-7B"
dataset_path = "dataset/medical_o1_sft_Chinese.json"
output_dir = "sft_lora_model_int8_output_new"

1
2
3
4
5

from datasets import load_dataset

dataset = load_dataset("json", data_files=dataset_path, split="train")
print(f"数据集一共有{len(dataset)}组数据")
print(f"数据集列：", dataset.column_names)

1

dataset = dataset.map(formatting_prompts_func, batched=True)

1
2
3

tokenizer = AutoTokenizer.from_pretrained(model_id_or_path, trust_remote_code=True)
if tokenizer.pad_token is None:
    tokenizer.pad_token = tokenizer.eos_token

1
2
3
4
5
6
7
8

bnb_config = BitsAndBytesConfig(load_in_8bit=True)

model = AutoModelForCausalLM.from_pretrained(
    model_id_or_path,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model

model = prepare_model_for_kbit_training(model)
lora_config = LoraConfig(
    r=16, lora_alpha=16,
    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
    lora_dropout=0, bias="none", task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

from trl import SFTTrainer
from transformers import TrainingArguments

training_arguments = TrainingArguments(
    output_dir=output_dir,
    per_device_train_batch_size=1,
    gradient_accumulation_steps=8,
    optim="adamw_8bit",
    save_steps=500,
    logging_steps=2,
    learning_rate=1e-4,
    num_train_epochs=1,
    max_grad_norm=0.3,
    lr_scheduler_type="cosine",
    weight_decay=0.01,
    warmup_ratio=0.03,
    bf16=True,
    report_to="wandb",
)

trainer = SFTTrainer(
    model=model, tokenizer=tokenizer,
    args=training_arguments,
    train_dataset=dataset,
    peft_config=lora_config,
    dataset_text_field="text",
    max_seq_length=2048,
    packing=False,
)

trainer.train()

1
2
3
4
5
6

trainer.save_model(output_dir)  # 保存LoRA adapter

metrics = train_result.metrics
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

1
2
3
4
5
6
7
8
9

# 使用nohup在后台运行
nohup accelerate launch train_on_single_gpu.py > training.log 2>&1 &

# 查看训练进度
tail -f training.log

# 停止训练
ps aux | grep "accelerate train_on_single_gpu.py"
kill <进程号>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

{
    "fp16": {
        "enabled": true,
        "loss_scale": 0,
        "loss_scale_window": 1000,
        "initial_scale_power": 16,
        "hysteresis": 2,
        "min_loss_scale": 1
    },
    "bf16": {
        "enabled": false
    },
    "optimizer": {
        "type": "AdamW",
        "params": {
            "lr": "auto",
            "betas": "auto",
            "eps": "auto",
            "weight_decay": "auto"
        }
    },
    "scheduler": {
        "type": "WarmupLR",
        "params": {
            "warmup_min_lr": "auto",
            "warmup_max_lr": "auto",
            "warmup_num_steps": "auto"
        }
    },
    "zero_optimization": {
        "stage": 2,
        "offload_optimizer": {
            "device": "none",
            "pin_memory": true
        },
        "allgather_partitions": true,
        "allgather_bucket_size": 2e8,
        "overlap_comm": true,
        "reduce_scatter": true,
        "reduce_bucket_size": 2e8,
        "contiguous_gradients": true
    },
    "gradient_accumulation_steps": "auto",
    "gradient_clipping": "auto",
    "train_batch_size": "auto",
    "train_micro_batch_size_per_gpu": "auto"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

compute_environment: LOCAL_MACHINE
distributed_type: DEEPSPEED

deepspeed_config:
  deepspeed_config_file: ds_config_zero2.json

gpu_ids: all
num_machines: 1
num_processes: 2        # GPU数量

rdzv_backend: static
same_network: true
use_cpu: false

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 1. 不要设置 device_map="auto"——交给accelerate处理
model = AutoModelForCausalLM.from_pretrained(
    model_id_or_path,
    # device_map="auto",  # 移除！accelerate会自动分配设备
    trust_remote_code=True
)

# 2. 在TrainingArguments中传入deepspeed配置
training_arguments = TrainingArguments(
    ...
    deepspeed=args.deepspeed,  # 传入DeepSpeed配置文件路径
    fp16=True,                 # 由DeepSpeed配置决定精度
    gradient_checkpointing=True,  # 梯度检查点节约显存
)

# 3. Wandb只在主进程初始化
if os.environ.get("LOCAL_RANK", "0") == "0":
    wandb.init(project="finetune_dsr1-ds")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 使用accelerate启动，指定DeepSpeed配置
accelerate launch \
    --config_file accelerate_config.yaml \
    --deepspeed_config_file ds_config_zero2.json \
    train_on_multi_gpu.py \
    --model_id_or_path DeepSeek-R1-Distill-Qwen-7B \
    --dataset_path dataset/medical_o1_sft_Chinese.json \
    --per_device_train_batch_size 1 \
    --gradient_accumulation_steps 8 \
    --learning_rate 1e-4

1
2

pip install unsloth
pip install bitsandbytes unsloth_zoo

1
2
3
4
5
6
7
8
9

from unsloth import FastLanguageModel
import torch

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/DeepSeek-R1-Distill-Llama-8B",
    max_seq_length=2048,
    dtype=None,          # None为自动检测，T4/V100用float16，Ampere+用bfloat16
    load_in_4bit=True,   # 启用4-bit量化
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

model = FastLanguageModel.get_peft_model(
    model,
    r=16,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    lora_alpha=16,
    lora_dropout=0,                         # 0是Unsloth优化过的
    bias="none",
    use_gradient_checkpointing="unsloth",   # 使用Unsloth优化的梯度检查点
    random_state=3407,
    use_rslora=False,
    loftq_config=None,
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

from trl import SFTTrainer
from transformers import TrainingArguments
from unsloth import is_bfloat16_supported

# 切换到训练模式
FastLanguageModel.for_training(model)

trainer = SFTTrainer(
    model=model, tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
    args=TrainingArguments(
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_steps=5,
        max_steps=60,
        learning_rate=2e-4,
        fp16=not is_bfloat16_supported(),
        bf16=is_bfloat16_supported(),
        logging_steps=1,
        optim="adamw_8bit",
        weight_decay=0.01,
        lr_scheduler_type="linear",
        seed=3407,
        output_dir="outputs",
    ),
)
trainer.train()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 切换到推理模式——比标准HuggingFace快2倍
FastLanguageModel.for_inference(model)

inputs = tokenizer([prompt], return_tensors="pt").to("cuda")
outputs = model.generate(
    input_ids=inputs.input_ids,
    attention_mask=inputs.attention_mask,
    max_new_tokens=1200,
    use_cache=True,
)
response = tokenizer.batch_decode(outputs)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 训练参数
per_device_train_batch_size = 2
gradient_accumulation_steps = 4    # 有效batch_size = 2 * 4 = 8
warmup_steps = 5
max_steps = 60                     # 仅训练60步（演示目的）
learning_rate = 2e-4
optim = "adamw_8bit"
weight_decay = 0.01
lr_scheduler_type = "linear"

# LoRA参数
r = 16
lora_alpha = 16
target_modules = ["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"]

1

{"input": "请问合同违约的法律后果是什么？", "output": "根据《民法典》第577条..."}

1
2
3
4
5
6
7
8

{
  "conversations": [
    {"from": "human", "value": "请问合同违约的法律后果是什么？"},
    {"from": "gpt", "value": "根据《民法典》第577条..."}
  ],
  "system": "",
  "tools": ""
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

import json

# 读取原始JSONL数据
json_data = []
with open('DISC-Law-SFT-Triplet-released.jsonl', 'r', encoding='utf-8') as file:
    for line in file:
        data = json.loads(line)
        json_data.append(data)

# 转换为目标格式
template = []
for data in json_data:
    conversation = [
        {"from": "human", "value": data["input"]},
        {"from": "gpt", "value": data["output"]}
    ]
    template.append({
        "conversations": conversation,
        "system": "",
        "tools": ""
    })

# 保存
with open('train_data_law.json', 'w', encoding='utf-8') as f:
    json.dump(template, f, ensure_ascii=False, indent=2)

1
2

learning_rate = 2e-4  # Unsloth医疗微调使用
learning_rate = 1e-4  # 单GPU/多GPU训练使用

1
2

per_device_train_batch_size = 1  # 显存紧张时
per_device_train_batch_size = 2  # Unsloth + 4bit量化时

1
2

gradient_accumulation_steps = 4  # Unsloth
gradient_accumulation_steps = 8  # 单GPU/多GPU

1
2

warmup_steps = 5       # 前5步线性增长学习率
warmup_ratio = 0.03    # 前3%的步数用于预热

1
2

lr_scheduler_type = "linear"   # 线性衰减到0
lr_scheduler_type = "cosine"   # 余弦退火

1

weight_decay = 0.01

1

max_grad_norm = 0.3

1
2

optim = "adamw_8bit"   # 使用bitsandbytes的8-bit AdamW
optim = "adamw_torch"  # 标准PyTorch AdamW

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

大模型微调 (Fine-Tuning)
├── 基础概念
│   ├── SFT监督微调
│   │   ├── 定义：在预训练模型上用标注数据继续训练
│   │   ├── 核心：labels = input_ids.clone()
│   │   └── 对比：微调 vs 预训练（数据量、计算量、学习率）
│   └── 数据格式
│       ├── Chat Template（apply_chat_template）
│       ├── 自定义Prompt Template
│       ├── EOS_TOKEN（必须添加）
│       └── 不同框架的数据格式要求
│
├── 参数高效微调 (PEFT)
│   ├── LoRA低秩适配
│   │   ├── 原理：W + BA, r << d
│   │   ├── 关键参数：r, lora_alpha, target_modules
│   │   └── 优势：可插拔、无推理延迟、节约显存
│   └── 量化技术
│       ├── 4-bit量化（nf4, double_quant）
│       ├── 8-bit量化
│       ├── QLoRA = 量化 + LoRA
│       └── prepare_model_for_kbit_training
│
├── 训练优化技术
│   ├── NEFT噪声嵌入
│   │   ├── 原理：向embedding添加均匀噪声
│   │   ├── 公式：alpha / sqrt(L * d)
│   │   └── 使用：neftune_noise_alpha=10
│   ├── Completion-Only训练
│   │   ├── Loss Mask机制
│   │   ├── DataCollatorForCompletionOnlyLM
│   │   └── 手动实现 vs 库函数
│   └── 超参数调优
│       ├── learning_rate (1e-5 ~ 5e-4)
│       ├── batch_size & gradient_accumulation
│       ├── lr_scheduler (cosine推荐)
│       ├── warmup & weight_decay
│       └── optim (adamw_8bit)
│
├── 训练框架
│   ├── 单GPU训练
│   │   ├── SFTTrainer + TrainingArguments
│   │   ├── 完整8步流程
│   │   └── accelerate launch
│   ├── 多GPU分布式训练
│   │   ├── DeepSpeed ZeRO-2
│   │   ├── accelerate_config.yaml
│   │   ├── ds_config_zero2.json
│   │   └── LOCAL_RANK环境变量
│   └── Unsloth框架
│       ├── FastLanguageModel.from_pretrained
│       ├── FastLanguageModel.get_peft_model
│       ├── for_training / for_inference
│       └── 2x速度、30%显存节约
│
└── 行业实战
    ├── 医疗领域
    │   ├── DeepSeek R1 + medical-o1-reasoning-SFT
    │   ├── 思考链（CoT）训练
    │   └── <think>...</think> 标签
    └── 法律领域
        ├── Qwen3 + DISC-Law-SFT
        ├── JSONL到LLaMA-Factory格式转换
        └── 数据预处理脚本