Upload 全流程.ipynb

全流程.ipynb

@@ -0,0 +1,536 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8b176d65-99f7-42a8-a6b6-4ec7ecceadf2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "基础包下载：\n",
+    "!pip install transformers sentencepiece google protobuf deepspeed peft datasets "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4702e6bb-8ade-4929-9981-f83b95d92606",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "设置huggingface镜像：\n",
+    "import os\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
+    "print(os.environ.get('HF_ENDPOINT'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f64e588-a7d3-4009-bc98-f45703781ae8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "autodl学术资源加速，在终端运行\n",
+    "source /etc/network_turbo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f4de3e11-6de0-4741-afa4-69dc73abd191",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#lfs 支持，用于git clone一些需要lfs的包\n",
+    "!apt-get update\n",
+    "!apt-get install git-lfs\n",
+    "!git lfs install"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a774d0a1-0582-443f-89c6-cd7f4a84966a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#下载好数据后，读取dna数据，分为训练集train和测试集test，默认已经shuffle\n",
+    "from datasets import load_dataset\n",
+    "dna_dataset = load_dataset('text', data_files='data/dna_1g.txt')['train'].train_test_split(test_size=0.05)\n",
+    "dna_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5243ba2d-1e95-4161-98d9-a403f7270c74",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dna_dataset[\"train\"][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5b5e607-31b1-433f-a094-2fad9e4bc472",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "前面这些数据集，就是常规的文本，一般就是当做预训练数据使用，而分类等下游微调任务，\n",
+    "一般都是包含标签的，多写成json或者csv的格式，这里也给出一个例子：\n",
+    "ft_dataset = load_dataset('json', data_files='data/dna_protein_my.json')\n",
+    "ft_dataset[\"train\"][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1ebc301-a222-4de9-be79-0150434f25f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "当然，如果数据集过大，我们只需要其中一部分，这个也是一个常见的需求，一般可以使用 Dataset.select()函数\n",
+    "from datasets import load_dataset, DatasetDict\n",
+    "dna_dataset_sample =  DatasetDict(\n",
+    "    {\n",
+    "        \"train\": dna_dataset[\"train\"].shuffle().select(range(50000)), \n",
+    "        \"valid\": dna_dataset[\"test\"].shuffle().select(range(500)),\n",
+    "        \"evla\": dna_dataset[\"test\"].shuffle().select(range(500))\n",
+    "\n",
+    "    }\n",
+    ")\n",
+    "dna_dataset_sample\n",
+    "可以看到，我们使用DatasetDict来直接构造datasets，先使用shuffle()来随机，然后使用select来选择前n个数据\n",
+    "select的参数为indices (list 或 range): 索引列表或范围对象，指明要选择哪些样本，\n",
+    "如dataset.select([0, 2, 4])就是选择1,3,5条记录"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ffbe618-7146-49c6-a8bd-f1d7e9f0ad4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "分享数据集到huggingface上面\n",
+    "dna_data.push_to_hub(\"org_name/your_dataset_name\", token=\"hf_yourtoken\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f8512c9e-c673-49db-a60a-f818a546852f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "从头训练一个基于BPE的DNA分词器\n",
+    "from tokenizers import (\n",
+    "    decoders,\n",
+    "    models,\n",
+    "    normalizers,\n",
+    "    pre_tokenizers,\n",
+    "    processors,\n",
+    "    trainers,\n",
+    "    Tokenizer,\n",
+    ")\n",
+    "from transformers import AutoTokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5937e169-ee42-44b4-9939-3792cde80ac5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "主分词器套分词算法\n",
+    "tokenizer = Tokenizer(models.BPE())\n",
+    "#预处理，ByteLevel就是按UTF-8分词，use_regex=False,空格当成一般字符串\n",
+    "tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False) \n",
+    "训练器，生成词表合并规则，词表大小3w\n",
+    "trainer1 = trainers.BpeTrainer(vocab_size=30000, special_tokens=[\"<|endoftext|>\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a7249164-b5f7-4a6f-9b2d-a30ae5a3e8b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "用DNA数据训练\n",
+    "tokenizer.train([\"../01-data_env/data/dna_1g.txt\"], trainer=trainer1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d3ddbec-ef6f-4d46-a82c-3b6e1eeacb74",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "encode执行分词并转换为ID\n",
+    "encoding = tokenizer.encode(\"TGGCGTGAACCCGGGATCGGG\")\n",
+    "print(encoding.tokens)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5539bc51-0d7d-4a31-a54a-597483b9861f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#save简单保存\n",
+    "tokenizer.save(\"dna_bpe_dict.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ecfed110-8840-4703-a659-b2d4a4d11f7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#然后我们可以使用from_file() 方法从该文件里重新加载 Tokenizer ：\n",
+    "new_tokenizer = Tokenizer.from_file(\"dna_bpe_dict.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15611626-c431-4f32-8f75-b3c2a6a67138",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#要在 hf Transformers中使用这个标记器，我们必须将它包裹在一个 PreTrainedTokenizerFast中\n",
+    "from transformers import GPT2TokenizerFast\n",
+    "dna_tokenizer = GPT2TokenizerFast(tokenizer_object=new_tokenizer)\n",
+    "#save_pretrained完整、规范地保存到磁盘，包含以下几个关键文件：\n",
+    "#1.xx.json是一个字典，映射了Token字符串到唯一ID\n",
+    "#2.merges.txt记录了BPE训练过程中所有的合并操作\n",
+    "#3.special_tokens_map.json 和 tokenizer_config.json这些是配置文件，定义了分词器的各种设置和行为。\n",
+    "#例如：哪些token是特殊token（如填充符<pad>、未知符<unk>、句首<s>），模型名称、最大长度等。\n",
+    "#这保证了分词器在不同环境中使用时的行为一致性。\n",
+    "dna_tokenizer.save_pretrained(\"dna_bpe_dict\")\n",
+    "#dna_tokenizer.push_to_hub(\"dna_bpe_dict_1g\", organization=\"dnagpt\", use_auth_token=\"hf_*****\") "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2624e2a7-7204-4c80-8806-74cb52ad11a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#save_pretrained 的标准逆操作，自动加载并实例化一个之前保存好的分词器。\n",
+    "tokenizer_new = AutoTokenizer.from_pretrained('dna_bpe_dict')\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c93cd0f4-4b4b-4a67-b35d-668f3b920806",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer_new.tokenize(\"TGGCGTGAACCCGGGATCGGG\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "768a983c-fdc4-4f67-90e4-6ff164e6c029",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "从头训练基于GPT2的DNA大模型\n",
+    "max_length = 256 #最大输入长度\n",
+    "#config加载并修改GPT2参数适配分词器\n",
+    "config = AutoConfig.from_pretrained(\n",
+    "    \"gpt2\",\n",
+    "    vocab_size=len(tokenizer),#标准的GPT-2是在英文文本上训练的，它的词汇表大小是固定的（比如50257），我们这个是3w\n",
+    "    n_ctx=max_length, #上下文长度（Context length），即模型一次能处理的最大Token数量\n",
+    "    bos_token_id=tokenizer.bos_token_id,#开始\n",
+    "    eos_token_id=tokenizer.eos_token_id,#停止\n",
+    ")\n",
+    "model = GPT2LMHeadModel(config) #权重初始化，从头预训练"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c002b619-07ca-41fc-b6c0-abfd17676934",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 1. 加载数据\n",
+    "raw_dataset = load_dataset('text',  data_files=\"../01-data_env/data/dna_1g.txt\")\n",
+    "dna_dataset = load_dataset('text', data_files='data/dna_1g.txt')['train'].train_test_split(test_size=0.05)\n",
+    "\n",
+    "# 2. （encode详细定义版）truncation=True过长截断, padding='max_length'过短填充到256, 界限是max_length=max_length=256\n",
+    "def tokenize_function(examples):\n",
+    "    return tokenizer(examples['text'], truncation=True, padding='max_length', max_length=max_length)\n",
+    "#tokenizer的作用：\n",
+    "#输出input_ids一个二维列表（List[List[int]]），里面是填充和截断后的Token ID序列。\n",
+    "#例如：[[105, 206, 307, ..., 0, 0, 0], [408, 509, 0, 0, ...], ...]\n",
+    "#attention_mask： 同样重要的配套输出。一个与 input_ids 形状相同的二维列表，但里面全是0和1。\n",
+    "#1 表示这个位置是真实的Token。\n",
+    "#0 表示这个位置是填充的Token ([PAD])\n",
+    "\n",
+    "# 3. 对数据集应用分词函数，移除原始文本text\n",
+    "tokenized_datasets = dna_dataset.map(tokenize_function, batched=True, remove_columns=['text'], num_proc=15)  # 设置为你�� CPU 核心数或根据需要调整\n",
+    "\n",
+    "# 4. 创建一个数据收集器，用于动态填充和遮蔽，tokenizer=tokenizer_new指定用于编码和解码的分词器对象\n",
+    "#上一步取到原始的 input_ids 后，会将这些数据交给 data_collator 函数，将它们处理成真正的 (inputs, labels) 对\n",
+    "data_collator = DataCollatorForLanguageModeling(\n",
+    "    tokenizer=tokenizer_new, mlm=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ccd72d9-f4a2-4792-8176-96afee553d27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "开始训练\n",
+    "run_path = \"gpt2_run\"\n",
+    "train_epoches = 5\n",
+    "batch_size = 10\n",
+    "\n",
+    "#TrainingArguments定义训练的“规则”，如：练多久、怎么练、在哪保存\n",
+    "training_args = TrainingArguments(\n",
+    "        output_dir=run_path,#指定所有输出结果的保存目录。这包括最终的模型、训练过程中的检查点（checkpoints）、日志和评估结果。\n",
+    "        overwrite_output_dir=True,#已有就覆盖\n",
+    "        num_train_epochs=train_epoches,#训练几次\n",
+    "        per_device_train_batch_size=batch_size,#每个GPU批次大小，每个10，俩GPU一共20\n",
+    "        save_steps=2000,\n",
+    "        save_total_limit=2,#每训练2000步就自动保存一个检查点（checkpoint），但只保留最新的2个。\n",
+    "        prediction_loss_only=True,#在评估（evaluation）时只计算损失（loss），而不计算其他指标（如准确率）\n",
+    "        fp16=True, #v100没法用\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,#model = GPT2LMHeadModel(config)\n",
+    "    args=training_args,#上面那个\n",
+    "    train_dataset=tokenized_datasets[\"train\"],\n",
+    "    eval_dataset=tokenized_datasets[\"test\"],\n",
+    "    data_collator=data_collator,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ff9c5a7-ce64-4bd6-92dd-7eaaa8b714c3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#训练完成后\n",
+    "import math\n",
+    "eval_results = trainer.evaluate()#使用上面创建trainer时给的测试集测试\n",
+    "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")\n",
+    "#困惑度=eval_results['eval_loss'计算损失--math.exp变成指数函数--.2f结果保留两位小数的浮点数\n",
+    "#困惑度是模型在预测下一个词时，平均面临的选择不确定性有多大"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a6bced4-2c0b-4ae1-911b-3a0eae83f202",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#上传模型\n",
+    "model.push_to_hub(\"dna_gpt2_v0\", organization=\"dnagpt\", use_auth_token=\"hf_*******\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20ab5784-8488-4991-8f16-85343d766baa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#训练完成后，我们就可以直接使用这个模型：\n",
+    "from transformers import AutoTokenizer, AutoModel\n",
+    "import torch\n",
+    "model = AutoModel.from_pretrained('dna_gpt2_v0')\n",
+    "model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1ae12f89-7887-4a71-8edf-a74982f0c2c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#应用1：取特征比如是不是启动子pt\n",
+    "from transformers import AutoTokenizer, AutoModel\n",
+    "tokenizer = AutoTokenizer.from_pretrained('dna_bpe_dict')\n",
+    "tokenizer.tokenize(\"GAGCACATTCGCCTGCGTGCGCACTCACACACACGTTCAAAAAGAGTCCATTCGATTCTGGCAGTAG\")\n",
+    "#result: [G','AGCAC','ATTCGCC',....]\n",
+    "\n",
+    "#我认为tokenizer.encode是输出token ID，\n",
+    "#tokenizer.tokenize输出人类可以阅读的分词结果，\n",
+    "#tokenizer(dna)输出ID和mask用于后续步骤\n",
+    "model = AutoModel.from_pretrained('dna_gpt2_v0')\n",
+    "import torch\n",
+    "dna = \"ACGTAGCATCGGATCTATCTATCGACACTTGGTTATCGATCTACGAGCATCTCGTTAGC\"\n",
+    "inputs = tokenizer(dna, return_tensors = 'pt')#指定返回 PyTorch张量（pytorch tensor)\n",
+    "print(inputs)#输入数据并分词\n",
+    "\n",
+    "#输出用模型分析的结果\n",
+    "outputs = model(inputs[\"input_ids\"])\n",
+    "\n",
+    "#提取特征：对一条DNA序列所有Token的隐藏状态向量求平均值，\n",
+    "#从而得到一个能够代表整条序列的、固定维度的嵌入式表示（Embedding）。\n",
+    "hidden_states = outputs.last_hidden_state # [使用最后一层，1批次大小,序列长度多少个token, 768隐藏层维度] \n",
+    "\n",
+    "# embedding with mean pooling\n",
+    "embedding_mean = torch.mean(hidden_states[0], dim=0)#通过索引 [0]，我们取出了批次中第一条，dim=0 指定了沿着哪个维度进行求平均。\n",
+    "#这里 dim=0 指的是沿着第0个维度（即序列长度维度，19个Token）进行���缩。\n",
+    "print(embedding_mean.shape) # expect to be 768\n",
+    "\n",
+    "# embedding with max pooling\n",
+    "embedding_max = torch.max(hidden_states[0], dim=0)[0]\n",
+    "print(embedding_max.shape) # expect to be 768\n",
+    "\n",
+    "# embedding with first token\n",
+    "embedding_first_token = hidden_states[0][0]\n",
+    "print(embedding_first_token.shape) # expect to be 768"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4ffacd74-15cb-4789-9c9b-d970be3915cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#获得embedding后开始分类，这个例子是线性全连接\n",
+    "import numpy as np\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.linear_model import LogisticRegression\n",
+    "from sklearn.metrics import accuracy_score\n",
+    "from transformers import GPT2Tokenizer, GPT2Model\n",
+    "import torch\n",
+    "\n",
+    "X = np.array(embedding_mean) # 上面得到的embedding_mean/max/first_token，将列表转为NumPy数组，形状 (n_samples样本数, 768)\n",
+    "y = np.array(labels)\n",
+    "\n",
+    "# 划分训练集和测试集\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
+    "\n",
+    "# 训练一个逻辑回归分类器\n",
+    "clf = LogisticRegression(random_state=42, max_iter=1000)\n",
+    "clf.fit(X_train, y_train)\n",
+    "\n",
+    "# 评估\n",
+    "accuracy = clf.score(X_test, y_test)\n",
+    "print(f\"Logistic Regression Accuracy: {accuracy:.4f}\")\n",
+    "\n",
+    "# 或者使用SVM\n",
+    "svm_clf = SVC(kernel='linear', random_state=42) # 线性核通常效果就很好\n",
+    "svm_clf.fit(X_train, y_train)\n",
+    "svm_accuracy = svm_clf.score(X_test, y_test)\n",
+    "print(f\"SVM Accuracy: {svm_accuracy:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b731af8d-0f9d-496a-9008-3e96bc98d671",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#获得embedding后开始分类，这个例子是神经网络\n",
+    "import torch.nn as nn\n",
+    "import torch.optim as optim\n",
+    "\n",
+    "# 定义一个简单的神经网络分类器\n",
+    "class MLPClassifier(nn.Module):\n",
+    "    def __init__(self, input_dim=768, hidden_dim=256, num_classes=2):\n",
+    "        super().__init__()\n",
+    "        self.layers = nn.Sequential(\n",
+    "            nn.Linear(input_dim, hidden_dim),\n",
+    "            nn.ReLU(),\n",
+    "            nn.Dropout(0.2), # 防止过拟合\n",
+    "            nn.Linear(hidden_dim, num_classes)\n",
+    "        )\n",
+    "    \n",
+    "    def forward(self, x):\n",
+    "        # x 是输入的embedding，形状 [batch_size, 768]\n",
+    "        return self.layers(x)\n",
+    "\n",
+    "# 使用流程\n",
+    "model = MLPClassifier(num_classes=3) # 假设是3分类任务\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n",
+    "\n",
+    "# 假设 train_loader 是已经准备好的PyTorch DataLoader\n",
+    "for epoch in range(10):\n",
+    "    for batch_embeddings, batch_labels in train_loader:\n",
+    "        optimizer.zero_grad()\n",
+    "        outputs = model(batch_embeddings)\n",
+    "        loss = criterion(outputs, batch_labels)\n",
+    "        loss.backward()\n",
+    "        optimizer.step()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e0e65a6f-faeb-4333-80f5-219ac2e0211e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/root/autodl-tmp/dnagpt2/01-data_env/data\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bfc1b2f8-62a8-422e-9a83-14a56c19272e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}