Add train and test codes

test.py

@@ -0,0 +1,46 @@
+from transformers import(
+    EncoderDecoderModel,
+    PreTrainedTokenizerFast,
+    # XLMRobertaTokenizerFast,
+    BertJapaneseTokenizer,
+    BertTokenizerFast,
+)
+
+import pandas as pd
+csv_test = pd.read_csv('./output/ffac_full.csv')
+# csv_test = pd.read_csv('ffac_test.csv')
+
+import csv
+
+encoder_model_name = "cl-tohoku/bert-base-japanese-v2"
+decoder_model_name = "skt/kogpt2-base-v2"
+
+src_tokenizer = BertJapaneseTokenizer.from_pretrained(encoder_model_name)
+trg_tokenizer = PreTrainedTokenizerFast.from_pretrained(decoder_model_name)
+model = EncoderDecoderModel.from_pretrained("./dump/best_model")
+
+def main():
+    data_test = []
+    data_test_label = []
+    data_test_infer = []
+    for row in csv_test.itertuples():
+        data_test.append(row[1])
+        data_test_label.append(row[2])
+
+    for text in data_test:
+        embeddings = src_tokenizer(text, return_attention_mask=False, return_token_type_ids=False, return_tensors='pt')
+        embeddings = {k: v for k, v in embeddings.items()}
+        output = model.generate(**embeddings)[0, 1:-1]
+        result = trg_tokenizer.decode(output.cpu())
+        # print(result)
+        data_test_infer.append(result)
+    
+    rows = zip(data_test, data_test_infer, data_test_label)
+    with open('test_result.csv', 'w') as f:
+        writer = csv.writer(f)
+        writer.writerow(['text', 'inference', 'answer'])
+        for row in rows:
+            writer.writerow(row)
+
+if __name__ == "__main__":
+    main()

test_eval.ipynb

@@ -0,0 +1,183 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Inference"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import(\n",
+    "    EncoderDecoderModel,\n",
+    "    PreTrainedTokenizerFast,\n",
+    "    # XLMRobertaTokenizerFast,\n",
+    "    BertJapaneseTokenizer,\n",
+    "    BertTokenizerFast,\n",
+    ")\n",
+    "\n",
+    "import torch\n",
+    "import csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
+      "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
+      "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
+     ]
+    }
+   ],
+   "source": [
+    "encoder_model_name = \"cl-tohoku/bert-base-japanese-v2\"\n",
+    "decoder_model_name = \"skt/kogpt2-base-v2\"\n",
+    "\n",
+    "src_tokenizer = BertJapaneseTokenizer.from_pretrained(encoder_model_name)\n",
+    "trg_tokenizer = PreTrainedTokenizerFast.from_pretrained(decoder_model_name)\n",
+    "model = EncoderDecoderModel.from_pretrained(\"./dump/best_model\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'길가메시 토벌전'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "text = \"ギルガメッシュ討伐戦\"\n",
+    "# text = \"ギルガメッシュ討伐戦に行ってきます。一緒に行きましょうか？\"\n",
+    "\n",
+    "def translate(text_src):\n",
+    "    embeddings = src_tokenizer(text_src, return_attention_mask=False, return_token_type_ids=False, return_tensors='pt')\n",
+    "    embeddings = {k: v for k, v in embeddings.items()}\n",
+    "    output = model.generate(**embeddings)[0, 1:-1]\n",
+    "    text_trg = trg_tokenizer.decode(output.cpu())\n",
+    "    return text_trg\n",
+    "\n",
+    "print(translate(text))"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Evaluation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
+    "smoothie = SmoothingFunction().method4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Testing:   0%|          | 0/267 [00:00<?, ?it/s]/home/tikim/.local/lib/python3.8/site-packages/transformers/generation/utils.py:1288: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n",
+      "Testing: 100%|██████████| 267/267 [01:01<00:00,  4.34it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bleu score: 0.9619225967540574\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tqdm import tqdm\n",
+    "from statistics import mean\n",
+    "\n",
+    "bleu = []\n",
+    "f1 = []\n",
+    "\n",
+    "DATA_ROOT = './output'\n",
+    "FILE_JP_KO_TEST = 'ja_ko_test.csv'\n",
+    "FILE_FFAC_TEST = 'ffac_test.csv'\n",
+    "\n",
+    "with torch.no_grad(), open(f'{DATA_ROOT}/{FILE_FFAC_TEST}', 'r') as fd:\n",
+    "# with torch.no_grad(), open(f'{DATA_ROOT}/{FILE_JP_KO_TEST}', 'r') as fd:\n",
+    "    reader = csv.reader(fd)\n",
+    "    next(reader)\n",
+    "    datas = [row for row in reader]    \n",
+    "\n",
+    "    for data in tqdm(datas, \"Testing\"):\n",
+    "        input, label = data\n",
+    "        embeddings = src_tokenizer(input, return_attention_mask=False, return_token_type_ids=False, return_tensors='pt')\n",
+    "        embeddings = {k: v for k, v in embeddings.items()}\n",
+    "        with torch.no_grad():\n",
+    "            output = model.generate(**embeddings)[0, 1:-1]\n",
+    "        preds = trg_tokenizer.decode(output.cpu())\n",
+    "\n",
+    "        bleu.append(sentence_bleu([label.split()], preds.split(), weights=[1,0,0,0], smoothing_function=smoothie))\n",
+    "\n",
+    "print(f\"Bleu score: {mean(bleu)}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

training.ipynb

@@ -0,0 +1,261 @@
+{
+  "cells": [
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The primary codes below are based on [akpe12/JP-KR-ocr-translator-for-travel](https://github.com/akpe12/JP-KR-ocr-translator-for-travel)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TrHlPFqwFAgj"
+      },
+      "source": [
+        "## Import"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t-jXeSJKE1WM"
+      },
+      "outputs": [],
+      "source": [
+        "\n",
+        "from typing import Dict, List\n",
+        "import csv\n",
+        "import torch\n",
+        "from transformers import (\n",
+        "    EncoderDecoderModel,\n",
+        "    GPT2Tokenizer as BaseGPT2Tokenizer,\n",
+        "    PreTrainedTokenizer, BertTokenizerFast,\n",
+        "    PreTrainedTokenizerFast,\n",
+        "    DataCollatorForSeq2Seq,\n",
+        "    Seq2SeqTrainingArguments,\n",
+        "    AutoTokenizer,\n",
+        "    XLMRobertaTokenizerFast,\n",
+        "    BertJapaneseTokenizer,\n",
+        "    Trainer\n",
+        ")\n",
+        "from torch.utils.data import DataLoader\n",
+        "from transformers.models.encoder_decoder.modeling_encoder_decoder import EncoderDecoderModel\n",
+        "\n",
+        "# encoder_model_name = \"xlm-roberta-base\"\n",
+        "encoder_model_name = \"cl-tohoku/bert-base-japanese-v2\"\n",
+        "decoder_model_name = \"skt/kogpt2-base-v2\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nEW5trBtbykK"
+      },
+      "outputs": [],
+      "source": [
+        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
+        "# device = torch.device(\"cpu\")\n",
+        "device, torch.cuda.device_count()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5ic7pUUBFU_v"
+      },
+      "outputs": [],
+      "source": [
+        "class GPT2Tokenizer(PreTrainedTokenizerFast):\n",
+        "    def build_inputs_with_special_tokens(self, token_ids: List[int]) -> List[int]:\n",
+        "        return token_ids + [self.eos_token_id]        \n",
+        "\n",
+        "src_tokenizer = BertJapaneseTokenizer.from_pretrained(encoder_model_name)\n",
+        "trg_tokenizer = GPT2Tokenizer.from_pretrained(decoder_model_name, bos_token='</s>', eos_token='</s>', unk_token='<unk>',\n",
+        "  pad_token='<pad>', mask_token='<mask>')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DTf4U1fmFQFh"
+      },
+      "source": [
+        "## Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65L4O1c5FLKt"
+      },
+      "outputs": [],
+      "source": [
+        "class PairedDataset:\n",
+        "    def __init__(self, \n",
+        "        src_tokenizer: PreTrainedTokenizerFast, tgt_tokenizer: PreTrainedTokenizerFast,\n",
+        "        file_path: str\n",
+        "    ):\n",
+        "        self.src_tokenizer = src_tokenizer\n",
+        "        self.trg_tokenizer = tgt_tokenizer\n",
+        "        with open(file_path, 'r') as fd:\n",
+        "            reader = csv.reader(fd)\n",
+        "            next(reader)\n",
+        "            self.data = [row for row in reader]\n",
+        "\n",
+        "    def __getitem__(self, index: int) -> Dict[str, torch.Tensor]:\n",
+        "        src, trg = self.data[index]\n",
+        "        embeddings = self.src_tokenizer(src, return_attention_mask=False, return_token_type_ids=False)\n",
+        "        embeddings['labels'] = self.trg_tokenizer.build_inputs_with_special_tokens(self.trg_tokenizer(trg, return_attention_mask=False)['input_ids'])\n",
+        "\n",
+        "        return embeddings\n",
+        "\n",
+        "    def __len__(self):\n",
+        "        return len(self.data)\n",
+        "    \n",
+        "DATA_ROOT = './output'\n",
+        "FILE_FFAC_FULL = 'ffac_full.csv'\n",
+        "FILE_FFAC_TEST = 'ffac_test.csv'\n",
+        "# FILE_JA_KO_TRAIN = 'ja_ko_train.csv'\n",
+        "# FILE_JA_KO_TEST = 'ja_ko_test.csv'\n",
+        "\n",
+        "train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_FULL}')\n",
+        "eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_TEST}') \n",
+        "# train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TRAIN}')\n",
+        "# eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TEST}')        "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uCBiLouSFiZY"
+      },
+      "source": [
+        "## Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I7uFbFYJFje8"
+      },
+      "outputs": [],
+      "source": [
+        "model = EncoderDecoderModel.from_encoder_decoder_pretrained(\n",
+        "    encoder_model_name,\n",
+        "    decoder_model_name,\n",
+        "    pad_token_id=trg_tokenizer.bos_token_id,\n",
+        ")\n",
+        "model.config.decoder_start_token_id = trg_tokenizer.bos_token_id"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YFq2GyOAUV0W"
+      },
+      "outputs": [],
+      "source": [
+        "# for Trainer\n",
+        "import wandb\n",
+        "\n",
+        "collate_fn = DataCollatorForSeq2Seq(src_tokenizer, model)\n",
+        "wandb.init(project=\"fftr-poc1\", name='jbert+kogpt2')\n",
+        "\n",
+        "arguments = Seq2SeqTrainingArguments(\n",
+        "    output_dir='dump',\n",
+        "    do_train=True,\n",
+        "    do_eval=True,\n",
+        "    evaluation_strategy=\"epoch\",\n",
+        "    save_strategy=\"epoch\",\n",
+        "#     num_train_epochs=5,\n",
+        "    num_train_epochs=25,\n",
+        "#     per_device_train_batch_size=32,\n",
+        "    per_device_train_batch_size=64,\n",
+        "#     per_device_eval_batch_size=32,\n",
+        "    per_device_eval_batch_size=64,\n",
+        "    warmup_ratio=0.1,\n",
+        "    gradient_accumulation_steps=4,\n",
+        "    save_total_limit=5,\n",
+        "    dataloader_num_workers=1,\n",
+        "    fp16=True,\n",
+        "    load_best_model_at_end=True,\n",
+        "    report_to='wandb'\n",
+        ")\n",
+        "\n",
+        "trainer = Trainer(\n",
+        "    model,\n",
+        "    arguments,\n",
+        "    data_collator=collate_fn,\n",
+        "    train_dataset=train_dataset,\n",
+        "    eval_dataset=eval_dataset\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pPsjDHO5Vc3y"
+      },
+      "source": [
+        "## Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_T4P4XunmK-C"
+      },
+      "outputs": [],
+      "source": [
+        "# model = EncoderDecoderModel.from_encoder_decoder_pretrained(\"xlm-roberta-base\",  \"skt/kogpt2-base-v2\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7vTqAgW6Ve3J"
+      },
+      "outputs": [],
+      "source": [
+        "trainer.train()\n",
+        "\n",
+        "model.save_pretrained(\"dump/best_model\")"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "machine_shape": "hm",
+      "provenance": []
+    },
+    "gpuClass": "premium",
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.10"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}