\u8fd9\u662f\u5173\u4e8e\u591a\u6a21\u6001 AI \u7684\u5927\u578b\u7cfb\u5217\u6587\u7ae0\u4e2d\u7684\u7b2c 4 \u7bc7\u3002\u5728\u4e0a\u4e00\u7bc7\u6587\u7ae0\u4e2d\uff0c\u6211\u4eec\u8ba8\u8bba\u4e86\u591a\u6a21\u6001 RAG \u7cfb\u7edf\uff0c\u5b83\u53ef\u4ee5\u4ece\u4e0d\u540c\u7684\u6570\u636e\u6a21\u6001\uff08\u4f8b\u5982\u6587\u672c\u3001\u56fe\u50cf\u3001\u97f3\u9891\uff09\u4e2d\u68c0\u7d22\u548c\u5408\u6210\u4fe1\u606f\u3002\u5728\u90a3\u91cc\uff0c\u6211\u4eec\u770b\u5230\u4e86\u5982\u4f55\u4f7f\u7528 CLIP \u5b9e\u73b0\u8fd9\u6837\u7684\u7cfb\u7edf\u3002\u7136\u800c\uff0c\u8fd9\u79cd\u65b9\u6cd5\u7684\u4e00\u4e2a\u95ee\u9898\u662f\uff0c\u901a\u7528\u5d4c\u5165\u6a21\u578b\uff08\u5982 CLIP\uff09\u7684\u5411\u91cf\u641c\u7d22\u7ed3\u679c\u5728\u7279\u5b9a\u9886\u57df\u7684\u7528\u4f8b\u4e2d\u53ef\u80fd\u8868\u73b0\u4e0d\u4f73\u3002<\/p>\n
\u5728\u672c\u6587\u4e2d\uff0c\u6211\u5c06\u8ba8\u8bba\u5982\u4f55\u901a\u8fc7\u5fae\u8c03\u591a\u6a21\u6001\u5d4c\u5165\u6a21\u578b\u6765\u7f13\u89e3\u8fd9\u4e9b\u95ee\u9898\u3002<\/p>\n
\u591a\u6a21\u6001\u5d4c\u5165\u8868\u793a\u540c\u4e00\u5411\u91cf\u7a7a\u95f4\u4e2d\u7684\u591a\u79cd\u6570\u636e\u6a21\u6001\uff0c\u56e0\u6b64\u76f8\u4f3c\u7684\u6982\u5ff5\u4f4d\u4e8e\u540c\u4e00\u4f4d\u7f6e\u3002\u4e0b\u9762\u663e\u793a\u4e86\u4e00\u4e2a\u76f4\u89c2\u7684\u793a\u4f8b\uff0c\u5176\u4e2d\u8bed\u4e49\u76f8\u4f3c\u7684\u9879\u76ee\uff08\u4f8b\u5982\u72d7\u7684\u56fe\u7247\u53ca\u5176\u5bf9\u5e94\u7684\u6807\u9898\uff09\u5f88\u63a5\u8fd1\uff0c\u800c\u4e0d\u76f8\u4f3c\u7684\u9879\u76ee\uff08\u4f8b\u5982\u732b\u7684\u56fe\u7247\u548c\u63cf\u8ff0\u72d7\u7684\u6807\u9898\uff09\u76f8\u8ddd\u5f88\u8fdc\u3002<\/p>\n
CLIP \u662f\u4e00\u79cd\u6d41\u884c\u7684\u591a\u6a21\u6001\u5d4c\u5165\u6a21\u578b\uff0c\u5b83\u4f7f\u7528\u5bf9\u6bd4\u5b66\u4e60\u5728\u5927\u91cf\u56fe\u50cf-\u6807\u9898\u5bf9\u8bed\u6599\u5e93\u4e0a\u8fdb\u884c\u8bad\u7ec3\u3002 CLIP \u7684\u5173\u952e\u89c1\u89e3\u662f\uff0c\u8fd9\u79cd\u6a21\u578b\u53ef\u4ee5\u89e3\u9501\u96f6\u6837\u672c\u80fd\u529b\uff0c\u4f8b\u5982\u56fe\u50cf\u5206\u7c7b\u3001\u641c\u7d22\u548c\u5b57\u5e55 [1]\u3002<\/p>\n
\u8fd9\u91cc\u7684\u5176\u4e2d\u4e00\u4e2a\u9650\u5236\u662f CLIP \u7684\u96f6\u6837\u672c\u80fd\u529b\u53ef\u80fd\u65e0\u6cd5\u5f88\u597d\u5730\u8f6c\u79fb\u5230\u6d89\u53ca\u4e13\u4e1a\u4fe1\u606f\u7684\u9886\u57df\uff0c\u4f8b\u5982\u5efa\u7b51\u56fe\u7eb8\u3001\u533b\u5b66\u6210\u50cf\u548c\u6280\u672f\u672f\u8bed\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5fae\u8c03\u6765\u63d0\u9ad8 CLIP \u7684\u6027\u80fd\u3002<\/p>\n
\u5fae\u8c03\u6d89\u53ca\u901a\u8fc7\u989d\u5916\u7684\u8bad\u7ec3\u5c06\u6a21\u578b\u8c03\u6574\u5230\u7279\u5b9a\u7528\u4f8b\u3002\u8fd9\u5f88\u5f3a\u5927\uff0c\u56e0\u4e3a\u5b83\u4f7f\u6211\u4eec\u80fd\u591f\u5728\u73b0\u6709\u7684\u6700\u5148\u8fdb\u6a21\u578b\u7684\u57fa\u7840\u4e0a\u6784\u5efa\u529f\u80fd\u5f3a\u5927\u7684\u4e13\u7528\u6a21\u578b\uff0c\u5e76\u4e14\u6570\u636e\u91cf\u76f8\u5bf9\u8f83\u5c0f\u3002<\/p>\n
\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u5173\u952e\u6b65\u9aa4\u4f7f\u7528 CLIP \u5b9e\u73b0\u6b64\u76ee\u7684\u3002<\/p>\n
\u6211\u5c06\u5728\u5177\u4f53\u793a\u4f8b\u4e2d\u8ba8\u8bba\u6bcf\u4e2a\u6b65\u9aa4\u3002<\/p>\n
\u5728\u8fd9\u91cc\uff0c\u6211\u5c06\u9488\u5bf9\u6211\u7684 YouTube \u9891\u9053\u4e2d\u7684\u6807\u9898\u548c\u7f29\u7565\u56fe\u5fae\u8c03 CLIP\u3002\u6700\u7ec8\uff0c\u6211\u4eec\u5c06\u5f97\u5230\u4e00\u4e2a\u6a21\u578b\uff0c\u8be5\u6a21\u578b\u53ef\u4ee5\u83b7\u53d6\u6807\u9898-\u7f29\u7565\u56fe\u5bf9\u5e76\u8fd4\u56de\u76f8\u4f3c\u5ea6\u5f97\u5206\u3002\u8fd9\u53ef\u7528\u4e8e\u5b9e\u9645\u5e94\u7528\uff0c\u4f8b\u5982\u5c06\u6807\u9898\u521b\u610f\u4e0e\u73b0\u6709\u7f29\u7565\u56fe\u5339\u914d\u6216\u5bf9\u7f29\u7565\u56fe\u5e93\u8fdb\u884c\u641c\u7d22\u3002<\/p>\n
\u3001\u548c\u5206\u522b\u5728 GitHub \u548c Hugging Face Hub \u4e0a\u514d\u8d39\u63d0\u4f9b\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528\u6b64\u4ee3\u7801\u548c\u6570\u636e\u6765\u8bad\u7ec3\u81ea\u5df1\u7684\u6a21\u578b\u3002\u5982\u679c\u4f60\u6700\u7ec8\u4f7f\u7528\u6b64\u6570\u636e\u96c6\u53d1\u5e03\u4efb\u4f55\u4f5c\u54c1\uff0c\u8bf7\u5f15\u7528\u539f\u59cb\u6765\u6e90 \ud83d\ude42<\/p>\n
\u4efb\u4f55\u5fae\u8c03\u8fc7\u7a0b\u7684\u7b2c\u4e00\u6b65\uff08\u4e5f\u662f\u6700\u91cd\u8981\u7684\u4e00\u6b65\uff09\u90fd\u662f\u6570\u636e\u6536\u96c6\u3002\u5728\u8fd9\u91cc\uff0c\u6211\u901a\u8fc7\u4e24\u6b65\u6d41\u7a0b\u4ece\u6211\u7684\u9891\u9053\u4e2d\u63d0\u53d6\u4e86\u6807\u9898-\u7f29\u7565\u56fe\u5bf9\u3002<\/p>\n
\u9996\u5148\uff0c\u6211\u4f7f\u7528 YouTube \u7684\u641c\u7d22 API \u63d0\u53d6\u6211\u9891\u9053\u4e0a\u6240\u6709\u89c6\u9891\u7684\u89c6\u9891 ID\u3002\u5176\u6b21\uff0c\u6211\u4f7f\u7528 YouTube \u7684\u89c6\u9891 API \u63d0\u53d6\u6bcf\u4e2a\u957f\u89c6\u9891\uff08\u5373\u65f6\u957f\u8d85\u8fc7 3 \u5206\u949f\uff09\u7684\u6807\u9898\u548c\u7f29\u7565\u56fe URL\u3002<\/p>\n
channel_id = 'UCa9gErQ9AE5jT2DZLjXBIdA' # my YouTube channel ID\npage_token = None # initialize page token\nurl = 'https:\/\/www.googleapis.com\/youtube\/v3\/search' # YouTube search API \n\n# extract video data across multiple search result pages\nvideo_id_list = []\n\nwhile page_token != 0:\n params = {\n \"key\": my_key, \n 'channelId': channel_id, \n 'part': [\"snippet\",\"id\"], \n 'order': \"date\", \n 'maxResults':50, \n 'pageToken': page_token\n }\n response = requests.get(url, params=params)\n\n for raw_item in dict(response.json())['items']:\n \n # only execute for youtube videos\n if raw_item['id']['kind'] != \"youtube#video\":\n continue\n\n # grab video ids\n video_id_list.append(raw_item['id']['videoId'])\n\n try:\n # grab next page token\n page_token = dict(response.json())['nextPageToken']\n except:\n # if no next page token kill while loop\n page_token = 0<\/code><\/pre>\n\u8bf7\u6ce8\u610f\uff0c\u4f60\u9700\u8981\u4e00\u4e2a YouTube API \u5bc6\u94a5\u6765\u8fd0\u884c\u4e0a\u8ff0 Python \u4ee3\u7801\uff0c\u53ef\u4ee5\u4f7f\u7528 \u521b\u5efa\u8be5\u5bc6\u94a5\u3002\u8981\u4f7f\u5176\u9002\u5e94\u4f60\u7684\u9891\u9053\uff0c\u53ea\u9700\u66f4\u6539 channel_id<\/code> \u53d8\u91cf\u3002<\/p>\n# extract video titles and thumbnails\nurl = \"https:\/\/www.googleapis.com\/youtube\/v3\/videos\"\nvideo_data_list = []\n\nfor video_id in video_id_list:\n\n params = {\n \"part\": [\"snippet\",\"contentDetails\"],\n \"id\": video_id, \n \"key\": my_key, \n }\n response = requests.get(url, params=params)\n \n raw_dict = dict(response.json())['items'][0]\n\n # only process videos longer than 3 minutes\n iso_duration = raw_dict['contentDetails'][\"duration\"]\n if parse_duration(iso_duration).total_seconds() < 180:\n continue\n \n # extract video data\n video_data = {}\n video_data['video_id'] = video_id\n video_data['title'] = raw_dict['snippet']['title']\n video_data['thumbnail_url'] = raw_dict['snippet']['thumbnails']['high']['url']\n\n # append data to list\n video_data_list.append(video_data)<\/code><\/pre>\n\u4f5c\u4e3a\u9644\u52a0\u6b65\u9aa4\uff0c\u6211\u521b\u5efa\u4e86\u8d1f\u7f29\u7565\u56fe\u6807\u9898\u5bf9\u3002\u6211\u4eec\u53ef\u4ee5\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u4f7f\u7528\u8fd9\u4e9b\u5bf9\uff0c\u4e0d\u4ec5\u53ef\u4ee5\u4e3a\u6a21\u578b\u63d0\u4f9b\u54ea\u4e9b\u5d4c\u5165\u5e94\u8be5\u9760\u8fd1\uff08\u5373\u6b63\u5bf9\uff09\u7684\u793a\u4f8b\uff0c\u8fd8\u53ef\u4ee5\u4e3a\u54ea\u4e9b\u5d4c\u5165\u5e94\u8be5\u8fdc\u79bb\uff08\u5373\u8d1f\u5bf9\uff09\u7684\u793a\u4f8b\u3002<\/p>\n
\u4e3a\u6b64\uff0c\u6211\u4f7f\u7528\u53e5\u5b50\u8f6c\u6362\u5668\u5e93\u8ba1\u7b97\u4e86\u6240\u6709\u53ef\u80fd\u7684\u6807\u9898\u5bf9\u4e4b\u95f4\u7684\u76f8\u4f3c\u6027\u3002\u7136\u540e\uff0c\u5bf9\u4e8e\u6bcf\u4e2a\u6b63\u5bf9\uff0c\u6211\u5c06\u6700\u4e0d\u76f8\u4f3c\u7684\u6807\u9898\u5339\u914d\u4e3a\u8d1f\u793a\u4f8b\u200b\u200b\uff08\u786e\u4fdd\u6ca1\u6709\u91cd\u590d\uff09\u3002<\/p>\n
# store data in dataframe\ndf = pd.DataFrame(video_data_list)\n\n# Load the model\nmodel = SentenceTransformer(\"all-mpnet-base-v2\")\n\n# Encode all titles\nembeddings = model.encode(df['title'].to_list())\n\n# compute similarities\nsimilarities = model.similarity(embeddings, embeddings)\n\n# match least JDs least similar to positive match as the negative match\nsimilarities_argsorted = np.argsort(similarities.numpy(), axis=1)\nnegative_pair_index_list = []\n\nfor i in range(len(similarities)):\n\n # Start with the smallest similarity index for the current row\n j = 0\n index = int(similarities_argsorted[i][j])\n\n # Ensure the index is unique\n while index in negative_pair_index_list:\n j += 1 # Move to the next smallest index\n index = int(similarities_argsorted[i][j]) # Fetch next smallest index\n\n negative_pair_index_list.append(index)\n\n# add negative pairs to df\ndf['title_neg'] = df['title'].iloc[negative_pair_index_list].values<\/code><\/pre>\n\u6700\u540e\uff0c\u6211\u521b\u5efa\u4e86\u4e00\u4e2a\u8bad\u7ec3-\u6709\u6548-\u6d4b\u8bd5\u5206\u5272\u5e76\u5c06\u6570\u636e\u96c6\u63a8\u9001\u5230 Hugging Face Hub\u3002<\/p>\n
# Shuffle the dataset\ndf = df.sample(frac=1, random_state=42).reset_index(drop=True)\n\n# Split into train, validation, and test sets\ntrain_frac = 0.7\nvalid_frac = 0.15\ntest_frac = 0.15\n\n# define train and validation size\ntrain_size = int(train_frac * len(df))\nvalid_size = int(valid_frac * len(df))\n\n# create train, validation, and test datasets\ndf_train = df[:train_size]\ndf_valid = df[train_size:train_size + valid_size]\ndf_test = df[train_size + valid_size:]\n\n# Convert the pandas DataFrames back to Hugging Face Datasets\ntrain_ds = Dataset.from_pandas(df_train)\nvalid_ds = Dataset.from_pandas(df_valid)\ntest_ds = Dataset.from_pandas(df_test)\n\n# Combine into a DatasetDict\ndataset_dict = DatasetDict({\n 'train': train_ds,\n 'valid': valid_ds,\n 'test': test_ds\n})<\/code><\/pre>\n# push data to hub\ndataset_dict.push_to_hub(\"shawhin\/yt-title-thumbnail-pairs\")<\/code><\/pre>\n2.2 \u9884\u5904\u7406\u8bad\u7ec3\u5bf9<\/h3>\n
\u867d\u7136\u6211\u4eec\u62e5\u6709\u5fae\u8c03\u6240\u9700\u7684\u6240\u6709\u6570\u636e\uff0c\u4f46\u5b83\u4ecd\u7136\u4e0d\u662f\u9002\u5408\u8bad\u7ec3\u7684\u683c\u5f0f\u3002\u66f4\u5177\u4f53\u5730\u8bf4\uff0c\u6211\u4eec\u9700\u8981\u5c06\u56fe\u50cf URL \u8f6c\u6362\u4e3a PIL \u56fe\u50cf\u5bf9\u8c61\uff0c\u5e76\u5c06\u6570\u636e\u7ec4\u7ec7\u6210\uff08\u951a\u70b9\u3001\u6b63\u3001\u8d1f\uff09\u4e09\u5143\u7ec4\uff0c\u5373\u7f29\u7565\u56fe\u3001\u5176\u5bf9\u5e94\u7684\u6807\u9898\u548c\u8d1f\u6807\u9898\u3002<\/p>\n
\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 Hugging Face Datasets \u5e93\u4ee5\u4ee5\u4e0b\u65b9\u5f0f\u5904\u7406\u6240\u6709\u4e09\u4e2a\u6570\u636e\u5206\u5272\uff08\u5373\u8bad\u7ec3\u3001\u9a8c\u8bc1\u548c\u6d4b\u8bd5\uff09\u3002<\/p>\n
from PIL import Image\n\n# load dataset\ndataset = load_dataset(\"shawhin\/yt-title-thumbnail-pairs\")\n\n# define preprocessing function\ndef preprocess(batch):\n \"\"\"\n Preprocessing data without augmentations for test set\n \"\"\"\n # get images from urls\n image_list = [Image.open(requests.get(url, stream=True).raw) \n for url in batch[\"thumbnail_url\"]]\n\n # return columns with standard names\n return {\n \"anchor\": image_list, \n \"positive\": batch[\"title\"], \n \"negative\": batch[\"title_neg\"]\n }\n\n# remove columns not relevant to training\ncolumns_to_remove = [col for col in dataset['train'].column_names \n if col not in ['anchor', 'positive', 'negative']]\n# apply transformations\ndataset = dataset.map(preprocess, batched=True, \n remove_columns=columns_to_remove)<\/code><\/pre>\n\u6211\u4eec\u5c06\u5217\u6309\uff08\u951a\u70b9\u3001\u6b63\u503c\u3001\u8d1f\u503c\uff09\u4e09\u5143\u7ec4\u6392\u5e8f\u5f88\u91cd\u8981\uff0c\u56e0\u4e3a\u8fd9\u662f\u6211\u4eec\u5728\u8bad\u7ec3\u671f\u95f4\u5c06\u4f7f\u7528\u7684\u635f\u5931\u51fd\u6570\u6240\u671f\u671b\u7684\u683c\u5f0f\uff08\u8fd9\u662f\u6211\u4ece\u75db\u82e6\u4e2d\u5b66\u5230\u7684\uff09\u3002<\/p>\n
2.3 \u5b9a\u4e49\u8bc4\u4f30<\/h3>\n
\u8bad\u7ec3\u6d89\u53ca\u4f18\u5316\u6a21\u578b\u7684\u53c2\u6570\u4ee5\u6700\u5c0f\u5316\u635f\u5931\u51fd\u6570\u3002\u7136\u800c\uff0c\u8fd9\u4e2a\u503c\uff08\u5373\u5bf9\u6bd4\u635f\u5931\uff09\u5f88\u5c11\u6709\u52a9\u4e8e\u8bc4\u4f30\u6a21\u578b\u5728\u4e0b\u6e38\u4efb\u52a1\uff08\u4f8b\u5982\u5c06\u6807\u9898\u4e0e\u7f29\u7565\u56fe\u5339\u914d\uff09\u4e0a\u7684\u8868\u73b0\u3002<\/p>\n
\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u4e00\u4e2a\u66f4\u6709\u6d1e\u5bdf\u529b\u7684\u91cf\u662f\u6a21\u578b\u5728\u591a\u4e2a\u5019\u9009\u4e2d\u6b63\u786e\u5339\u914d\u7ed9\u5b9a\u7f29\u7565\u56fe\u4e0e\u6b63\u786e\u6807\u9898\u7684\u80fd\u529b\u3002\u8fd9\u8868\u793a\u4e3a Recall@1<\/code>\u3002<\/p>\n\u6211\u4eec\u53ef\u4ee5\u5b9e\u73b0\u4e00\u4e2a\u4e0e Sentence Transformers<\/code> \u5e93\u517c\u5bb9\u7684\u8bc4\u4f30\u5668\u6765\u8ba1\u7b97\u8fd9\u4e2a\u6307\u6807\u3002\u7531\u4e8e\u4ee3\u7801\u5f88\u957f\uff0c\u6211\u4e0d\u4f1a\u5728\u8fd9\u91cc\u7c98\u8d34\u5b83\uff0c\u4f46\u597d\u5947\u7684\u8bfb\u8005\u53ef\u4ee5\u5728\u7684\u7b2c 12 \u5355\u5143\u4e2d\u627e\u5230\u5b83\u3002<\/p>\n# function to create new evaluator given data split\ndef create_recall_evaluator(set_name, k=1):\n \"\"\"\n Create triplet evaluator for \"train\", \"valid\", or \"test\" split\n \"\"\"\n\n return ImageTextRetrievalEvaluator(\n images=dataset[f\"{set_name}\"][\"anchor\"],\n texts=dataset[f\"{set_name}\"][\"positive\"],\n name=f\"yt-title-thumbnail-{set_name}\",\n k=k\n )\n\n# Create new evaluator with Recall@k\nevaluator_recall_train = create_recall_evaluator(\"train\", k=1)\nevaluator_recall_valid = create_recall_evaluator(\"valid\", k=1)\n\nprint(\"Train:\", evaluator_recall_train(model))\nprint(\"Valid:\", evaluator_recall_valid(model))\n\n# >> Train: {'yt-title-thumbnail-train_Recall@1': 0.660377358490566}\n# >> Valid: {'yt-title-thumbnail-valid_Recall@1': 0.6363636363636364}<\/code><\/pre>\n\u6211\u4eec\u53ef\u4ee5\u770b\u5230\uff0c\u8be5\u6a21\u578b\u5f00\u7bb1\u5373\u7528\uff0c\u6027\u80fd\u4e0d\u9519\uff0c66% \u7684\u65f6\u95f4\u5339\u914d\u6b63\u786e\u7684\u6807\u9898\u3002<\/p>\n
2.4 \u5fae\u8c03\u6a21\u578b<\/h3>\n
\u5728\u8bad\u7ec3\u6a21\u578b\u4e4b\u524d\uff0c\u6211\u4eec\u5fc5\u987b\u505a 3 \u4ef6\u5173\u952e\u7684\u4e8b\u60c5\u3002\u5373\uff0c\u9009\u62e9\u8981\u8bad\u7ec3\u7684\u53c2\u6570\u3001\u9009\u62e9\u635f\u5931\u51fd\u6570\u548c\u8bbe\u7f6e\u8d85\u53c2\u6570\u3002<\/p>\n
\n- \u53ef\u8bad\u7ec3\u53c2\u6570<\/li>\n<\/ul>\n
\u8be5\u9879\u76ee\u7684\u4e3b\u8981\u9650\u5236\u662f\u6211\u53ea\u53d1\u5e03\u4e86 76 \u4e2a YouTube \u89c6\u9891\uff08\u622a\u81f3\u64b0\u5199\u672c\u6587\u65f6\uff09\u3002\u901a\u8fc7\u9a8c\u8bc1\u548c\u6d4b\u8bd5\u62c6\u5206\uff0c\u53ea\u5269\u4e0b 53 \u4e2a\u793a\u4f8b\u53ef\u4f9b\u8bad\u7ec3\u3002<\/p>\n
\u7531\u4e8e\u6211\u4eec\u7684\u8bad\u7ec3\u6837\u672c\u975e\u5e38\u5c11\uff0c\u9650\u5236\u8bad\u7ec3\u53c2\u6570\u7684\u6570\u91cf\u662f\u4e2a\u597d\u4e3b\u610f\u3002\u5728\u672c\u4f8b\u4e2d\uff0c\u6211\u53ea\u8bad\u7ec3\u6a21\u578b\u7684\u6700\u7ec8\u6295\u5f71\u5c42\uff0c\u8be5\u5c42\u5c06\u6587\u672c\u548c\u56fe\u50cf\u5d4c\u5165\u6620\u5c04\u5230\u5171\u4eab\u5411\u91cf\u7a7a\u95f4\u4e2d\u3002\u603b\u5171\u7ea6\u6709 1M \u4e2a\u53c2\u6570\u3002<\/p>\n
# import model\nfrom sentence_transformers import SentenceTransformer\nmodel = SentenceTransformer(\"sentence-transformers\/clip-ViT-L-14\")\n\n# pick specific layers to train (note: you can add more layers to this list)\ntrainable_layers_list = ['projection']\n\n# Apply freezing configuration\nfor name, param in model.named_parameters():\n \n # freeze all params\n param.requires_grad = False\n\n # unfreeze layers in trainable_layers_list\n if any(layer in name for layer in trainable_layers_list):\n param.requires_grad = True<\/code><\/pre>\n# Count total and trainable parameters\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")\nprint(f\"% of trainable parameters: {100*trainable_params\/total_params:.2f}%\")\n\n# >> Total parameters: 427,616,513\n# >> Trainable parameters: 1,376,256\n# >> % of trainable parameters: 0.32%<\/code><\/pre>\n\n- \u635f\u5931\u51fd\u6570<\/li>\n<\/ul>\n
\u5728\u8fd9\u91cc\uff0c\u6211\u4f7f\u7528 Sentence Transformers \u5e93\u4e2d\u7684\uff08\u5728\u672c\u4f8b\u4e2d\uff0c\u5b83\u9002\u7528\u4e8e\u5355\u4e2a\u8d1f\u6837\u672c\uff09\u3002\u5b83\u7684\u5de5\u4f5c\u539f\u7406\u662f\u6700\u5927\u5316\u6b63\u6837\u672c\u5bf9\u4e4b\u95f4\u7684\u76f8\u4f3c\u6027\uff0c\u540c\u65f6\u6700\u5c0f\u5316\u8d1f\u6837\u672c\u5bf9\u4e4b\u95f4\u7684\u76f8\u4f3c\u6027\u3002\u4ee5\u4e0b\u662f\u5355\u4e2a\u8d1f\u6837\u672c\u60c5\u51b5\u7684\u635f\u5931\u51fd\u6570 [2]\u3002<\/p>\n
from sentence_transformers.losses import MultipleNegativesRankingLoss\n\n# define loss\nloss = MultipleNegativesRankingLoss(model)<\/code><\/pre>\n\n- \u8d85\u53c2\u6570<\/li>\n<\/ul>\n
\u5bf9\u4e8e\u8d85\u53c2\u6570\uff0c\u6211\u624b\u52a8\u5c1d\u8bd5\u4e86\u4e00\u4e9b\u9009\u62e9\uff0c\u5e76\u9009\u62e9\u4e86\u9a8c\u8bc1\u635f\u5931\u548c Recall@1<\/code> \u6027\u80fd\u6700\u4f73\u7684\u9009\u62e9\u3002\u4ee5\u4e0b\u662f\u6700\u7ec8\u7684\u9009\u62e9\u3002<\/p>\nfrom sentence_transformers import SentenceTransformerTrainingArguments\n\n# hyperparameters\nnum_epochs = 2\nbatch_size = 16\nlr = 1e-4\nfinetuned_model_name = \"clip-title-thumbnail-embeddings\"\n\ntrain_args = SentenceTransformerTrainingArguments(\n output_dir=f\"models\/{finetuned_model_name}\",\n num_train_epochs=num_epochs,\n per_device_train_batch_size=batch_size,\n per_device_eval_batch_size=batch_size,\n learning_rate=lr,\n # Evaluation settings\n eval_strategy=\"epoch\",\n eval_steps=1,\n logging_steps=1,\n)<\/code><\/pre>\n\u5b9a\u4e49\u597d\u635f\u5931\u548c\u8d85\u53c2\u6570\u540e\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 SentenceTransformersTrainer()<\/code> \u6765\u8bad\u7ec3\u6a21\u578b\u3002<\/p>\nfrom sentence_transformers import SentenceTransformerTrainer\n\ntrainer = SentenceTransformerTrainer(\n model=model,\n args=train_args,\n train_dataset=dataset[\"train\"],\n eval_dataset=dataset[\"valid\"],\n loss=loss,\n evaluator=[evaluator_recall_train, evaluator_recall_valid],\n)\ntrainer.train()<\/code><\/pre>\n\u6a21\u578b\u8bad\u7ec3\u662f\u4e00\u4e2a\u8fed\u4ee3\u8fc7\u7a0b\uff0c\u4f60\u53ef\u4ee5\u5728\u5176\u4e2d\u63a2\u7d22\u6570\u5341\u79cd\u6a21\u578b\uff0c\u4ee5\u9009\u62e9\u4e0d\u540c\u7684\u53ef\u8bad\u7ec3\u53c2\u6570\u3001\u635f\u5931\u51fd\u6570\u548c\u8d85\u53c2\u6570\u3002<\/p>\n
\u7136\u800c\uff0c\u6211\u5f3a\u70c8\u5efa\u8bae\u5c3d\u53ef\u80fd\u7b80\u5316\u8fd9\u4e9b\u5b9e\u9a8c\u3002\u5982\u679c\u4f60\u53d1\u73b0\u81ea\u5df1\u82b1\u4e86\u592a\u591a\u65f6\u95f4\u8c03\u6574\u8bad\u7ec3\u53c2\u6570\u4ee5\u4f7f\u6a21\u578b\u6536\u655b\uff0c\u90a3\u4e48\u4f60\u7684\u6570\u636e\u53ef\u80fd\u5b58\u5728\u6839\u672c\u6027\u9519\u8bef\uff08\u7ecf\u9a8c\u4e4b\u8c08\uff09\u3002<\/p>\n
2.5 \u8bc4\u4f30\u6a21\u578b<\/h3>\n
\u6700\u540e\u4e00\u6b65\uff0c\u6211\u4eec\u53ef\u4ee5\u8bc4\u4f30\u6a21\u578b\u5728\u6d4b\u8bd5\u96c6\u4e0a\u7684 Recall@1<\/code> \u5206\u6570\u3002\u8fd9\u4e9b\u6570\u636e\u672a\u7528\u4e8e\u8bad\u7ec3\u6216\u8d85\u53c2\u6570\u8c03\u6574\uff0c\u56e0\u6b64\u5b83\u4e3a\u6211\u4eec\u63d0\u4f9b\u4e86\u5bf9\u6a21\u578b\u7684\u516c\u6b63\u8bc4\u4f30\u3002<\/p>\nevaluator_recall_test = create_recall_evaluator(\"test\")\n\nprint(\"Train:\", evaluator_recall_train(model))\nprint(\"Valid:\", evaluator_recall_valid(model))\nprint(\"Test:\", evaluator_recall_test(model))\n\n# >> Train: {'yt-title-thumbnail-train_Recall@1': 0.8490566037735849}\n# >> Valid: {'yt-title-thumbnail-valid_Recall@1': 0.9090909090909091}\n# >> Test: {'yt-title-thumbnail-test_Recall@1': 0.75}<\/code><\/pre>\n\u6211\u4eec\u770b\u5230\u6a21\u578b\u5728\u6240\u6709\u4e09\u4e2a\u6570\u636e\u96c6\u4e0a\u8868\u73b0\u826f\u597d\uff0c\u6d4b\u8bd5\u96c6\u4e0a\u7684 Recall@1<\/code> \u4e3a 75%\u3002\u6362\u53e5\u8bdd\u8bf4\uff0c75% \u7684\u65f6\u95f4\u91cc\uff0c\u8be5\u6a21\u578b\u80fd\u591f\u6b63\u786e\u5730\u5c06\u7ed9\u5b9a\u7684\u7f29\u7565\u56fe\u4e0e\u5176\u539f\u59cb\u6807\u9898\u5339\u914d\u3002\u6b64\u5916\uff0c\u9a8c\u8bc1\u6570\u636e\u96c6\u7684\u53ec\u56de\u7387\u4e5f\u589e\u52a0\u4e86 27%\uff01<\/p>\n3\u3001\u63a5\u4e0b\u6765\u662f\u4ec0\u4e48\uff1f<\/h2>\n
\u591a\u6a21\u6001\u5d4c\u5165\u6a21\u578b\uff08\u5982 CLIP\uff09\u89e3\u9501\u4e86\u65e0\u6570 0-shot \u7528\u4f8b\uff0c\u4f8b\u5982\u56fe\u50cf\u5206\u7c7b\u548c\u68c0\u7d22\u3002\u5728\u8fd9\u91cc\uff0c\u6211\u4eec\u770b\u5230\u4e86\u5982\u4f55\u5fae\u8c03\u8fd9\u6837\u7684\u6a21\u578b\u4ee5\u4f7f\u5176\u9002\u5e94\u4e13\u95e8\u7684\u9886\u57df\uff08\u5373\u6211\u7684 YouTube \u6807\u9898\u548c\u7f29\u7565\u56fe\uff09\u3002<\/p>\n
\u867d\u7136 CLIP \u6309\u7167\u4eca\u5929\u7684\u6807\u51c6\u6765\u8bf4\u662f\u4e00\u4e2a\u5c0f\u6a21\u578b\uff08\u7ea6 5 \u4ebf\u4e2a\u53c2\u6570\uff09\uff0c\u800c\u4e14\u6211\u4eec\u7684\u8bad\u7ec3\u6570\u636e\u96c6\u5f88\u5c0f\uff0c\u4f46\u6700\u7ec8\u6a21\u578b\u4ecd\u7136\u5728\u8fd9\u4e2a\u4efb\u52a1\u4e0a\u8868\u73b0\u51fa\u8272\u3002\u8fd9\u51f8\u663e\u4e86\u5fae\u8c03\u7684\u5a01\u529b\u3002<\/p>\n
\n","protected":false},"excerpt":{"rendered":"\u8fd9\u662f\u5173\u4e8e\u591a\u6a21\u6001 AI \u7684\u5927\u578b\u7cfb\u5217\u6587\u7ae0\u4e2d\u7684\u7b2c 4 \u7bc7\u3002\u5728\u4e0a\u4e00\u7bc7\u6587\u7ae0\u4e2d\uff0c\u6211\u4eec\u8ba8\u8bba\u4e86\u591a\u6a21\u6001 RAG \u7cfb\u7edf\uff0c\u5b83\u53ef\u4ee5\u4ece\u4e0d\u540c\u7684\u6570\u636e\u6a21\u6001\uff08\u4f8b\u5982\u6587\u672c\u3001\u56fe\u50cf\u3001\u97f3\u9891\uff09\u4e2d\u68c0\u7d22\u548c\u5408\u6210\u4fe1\u606f\u3002\u5728\u90a3\u91cc\uff0c\u6211\u4eec\u770b\u5230\u4e86\u5982\u4f55\u4f7f\u7528 CLIP \u5b9e\u73b0\u8fd9\u6837\u7684\u7cfb\u7edf\u3002\u7136\u800c\uff0c\u8fd9\u79cd\u65b9\u6cd5\u7684\u4e00\u4e2a\u95ee\u9898\u662f\uff0c\u901a\u7528\u5d4c\u5165\u6a21\u578b\uff08\u5982 CLIP\uff09\u7684\u5411\u91cf\u641c\u7d22\u7ed3\u679c\u5728\u7279\u5b9a\u9886\u57df\u7684\u7528\u4f8b\u4e2d\u53ef\u80fd\u8868\u73b0\u4e0d\u4f73\u3002 \u5728\u672c\u6587\u4e2d\uff0c\u6211\u5c06\u8ba8\u8bba\u5982\u4f55\u901a\u8fc7\u5fae\u8c03\u591a\u6a21\u6001\u5d4c\u5165\u6a21\u578b\u6765\u7f13\u89e3\u8fd9\u4e9b\u95ee\u9898\u3002 \u591a\u6a21\u6001\u5d4c\u5165\u8868\u793a\u540c\u4e00\u5411\u91cf\u7a7a\u95f4\u4e2d\u7684\u591a\u79cd\u6570\u636e\u6a21\u6001\uff0c\u56e0\u6b64\u76f8\u4f3c\u7684\u6982\u5ff5\u4f4d\u4e8e\u540c\u4e00\u4f4d\u7f6e\u3002\u4e0b\u9762\u663e\u793a\u4e86\u4e00\u4e2a\u76f4\u89c2\u7684\u793a\u4f8b\uff0c\u5176\u4e2d\u8bed\u4e49\u76f8\u4f3c\u7684\u9879\u76ee\uff08\u4f8b\u5982\u72d7\u7684\u56fe\u7247\u53ca\u5176\u5bf9\u5e94\u7684\u6807\u9898\uff09\u5f88\u63a5\u8fd1\uff0c\u800c\u4e0d\u76f8\u4f3c\u7684\u9879\u76ee\uff08\u4f8b\u5982\u732b\u7684\u56fe\u7247\u548c\u63cf\u8ff0\u72d7\u7684\u6807\u9898\uff09\u76f8\u8ddd\u5f88\u8fdc\u3002 CLIP \u662f\u4e00\u79cd\u6d41\u884c\u7684\u591a\u6a21\u6001\u5d4c\u5165\u6a21\u578b\uff0c\u5b83\u4f7f\u7528\u5bf9\u6bd4\u5b66\u4e60\u5728\u5927\u91cf\u56fe\u50cf-\u6807\u9898\u5bf9\u8bed\u6599\u5e93\u4e0a\u8fdb\u884c\u8bad\u7ec3\u3002 CLIP \u7684\u5173\u952e\u89c1\u89e3\u662f\uff0c\u8fd9\u79cd\u6a21\u578b\u53ef\u4ee5\u89e3\u9501\u96f6\u6837\u672c\u80fd\u529b\uff0c\u4f8b\u5982\u56fe\u50cf\u5206\u7c7b\u3001\u641c\u7d22\u548c\u5b57\u5e55 [1]\u3002 \u8fd9\u91cc\u7684\u5176\u4e2d\u4e00\u4e2a\u9650\u5236\u662f CLIP \u7684\u96f6\u6837\u672c\u80fd\u529b\u53ef\u80fd\u65e0\u6cd5\u5f88\u597d\u5730\u8f6c\u79fb\u5230\u6d89\u53ca\u4e13\u4e1a\u4fe1\u606f\u7684\u9886\u57df\uff0c\u4f8b\u5982\u5efa\u7b51\u56fe\u7eb8\u3001\u533b\u5b66\u6210\u50cf\u548c\u6280\u672f\u672f\u8bed\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5fae\u8c03\u6765\u63d0\u9ad8 CLIP \u7684\u6027\u80fd\u3002 1\u3001CLIP\u5fae\u8c03\u6982\u8ff0 \u5fae\u8c03\u6d89\u53ca\u901a\u8fc7\u989d\u5916\u7684\u8bad\u7ec3\u5c06\u6a21\u578b\u8c03\u6574\u5230\u7279\u5b9a\u7528\u4f8b\u3002\u8fd9\u5f88\u5f3a\u5927\uff0c\u56e0\u4e3a\u5b83\u4f7f\u6211\u4eec\u80fd\u591f\u5728\u73b0\u6709\u7684\u6700\u5148\u8fdb\u6a21\u578b\u7684\u57fa\u7840\u4e0a\u6784\u5efa\u529f\u80fd\u5f3a\u5927\u7684\u4e13\u7528\u6a21\u578b\uff0c\u5e76\u4e14\u6570\u636e\u91cf\u76f8\u5bf9\u8f83\u5c0f\u3002 \u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u5173\u952e\u6b65\u9aa4\u4f7f\u7528 CLIP \u5b9e\u73b0\u6b64\u76ee\u7684\u3002 \u6536\u96c6\u6587\u672c-\u56fe\u50cf\u8bad\u7ec3\u5bf9 \u9884\u5904\u7406\u8bad\u7ec3\u6570\u636e \u5b9a\u4e49\u8bc4\u4f30 \u5fae\u8c03\u6a21\u578b \u8bc4\u4f30\u6a21\u578b \u6211\u5c06\u5728\u5177\u4f53\u793a\u4f8b\u4e2d\u8ba8\u8bba\u6bcf\u4e2a\u6b65\u9aa4\u3002 2\u3001\u7528YouTube\u6807\u9898\u548c\u7f29\u7565\u56fe\u5fae\u8c03 CLIP \u5728\u8fd9\u91cc\uff0c\u6211\u5c06\u9488\u5bf9\u6211\u7684 YouTube \u9891\u9053\u4e2d\u7684\u6807\u9898\u548c\u7f29\u7565\u56fe\u5fae\u8c03 CLIP\u3002\u6700\u7ec8\uff0c\u6211\u4eec\u5c06\u5f97\u5230\u4e00\u4e2a\u6a21\u578b\uff0c\u8be5\u6a21\u578b\u53ef\u4ee5\u83b7\u53d6\u6807\u9898-\u7f29\u7565\u56fe\u5bf9\u5e76\u8fd4\u56de\u76f8\u4f3c\u5ea6\u5f97\u5206\u3002\u8fd9\u53ef\u7528\u4e8e\u5b9e\u9645\u5e94\u7528\uff0c\u4f8b\u5982\u5c06\u6807\u9898\u521b\u610f\u4e0e\u73b0\u6709\u7f29\u7565\u56fe\u5339\u914d\u6216\u5bf9\u7f29\u7565\u56fe\u5e93\u8fdb\u884c\u641c\u7d22\u3002 \u3001\u548c\u5206\u522b\u5728 GitHub \u548c Hugging Face Hub \u4e0a\u514d\u8d39\u63d0\u4f9b\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528\u6b64\u4ee3\u7801\u548c\u6570\u636e\u6765\u8bad\u7ec3\u81ea\u5df1\u7684\u6a21\u578b\u3002\u5982\u679c\u4f60\u6700\u7ec8\u4f7f\u7528\u6b64\u6570\u636e\u96c6\u53d1\u5e03\u4efb\u4f55\u4f5c\u54c1\uff0c\u8bf7\u5f15\u7528\u539f\u59cb\u6765\u6e90 \ud83d\ude42 2.1 \u6536\u96c6\u6587\u672c-\u56fe\u50cf\u8bad\u7ec3\u5bf9 \u4efb\u4f55\u5fae\u8c03\u8fc7\u7a0b\u7684\u7b2c\u4e00\u6b65\uff08\u4e5f\u662f\u6700\u91cd\u8981\u7684\u4e00\u6b65\uff09\u90fd\u662f\u6570\u636e\u6536\u96c6\u3002\u5728\u8fd9\u91cc\uff0c\u6211\u901a\u8fc7\u4e24\u6b65\u6d41\u7a0b\u4ece\u6211\u7684\u9891\u9053\u4e2d\u63d0\u53d6\u4e86\u6807\u9898-\u7f29\u7565\u56fe\u5bf9\u3002 \u9996\u5148\uff0c\u6211\u4f7f\u7528 YouTube \u7684\u641c\u7d22 API \u63d0\u53d6\u6211\u9891\u9053\u4e0a\u6240\u6709\u89c6\u9891\u7684\u89c6\u9891 […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[13],"tags":[],"class_list":["post-53773","post","type-post","status-publish","format-standard","hentry","category-ai"],"_links":{"self":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts\/53773","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/comments?post=53773"}],"version-history":[{"count":0,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts\/53773\/revisions"}],"wp:attachment":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/media?parent=53773"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/categories?post=53773"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/tags?post=53773"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}