{"id":53738,"date":"2025-02-16T08:58:13","date_gmt":"2025-02-16T00:58:13","guid":{"rendered":"https:\/\/fwq.ai\/blog\/53738\/"},"modified":"2025-02-16T08:58:13","modified_gmt":"2025-02-16T00:58:13","slug":"dspy-image%ef%bc%9a%e8%a7%86%e8%a7%89%e6%a8%a1%e5%9e%8b%e6%94%af%e6%8c%81","status":"publish","type":"post","link":"https:\/\/fwq.ai\/blog\/53738\/","title":{"rendered":"DSPy.Image\uff1a\u89c6\u89c9\u6a21\u578b\u652f\u6301"},"content":{"rendered":"

DSPy \u6700\u8fd1\u5728\u6d4b\u8bd5\u7248\u4e2d\u589e\u52a0\u4e86\u5bf9 VLM \u7684\u652f\u6301\u3002\u672c\u6587\u4ecb\u7ecd\u4f7f\u7528 DSPy \u4ece\u56fe\u50cf\u4e2d\u63d0\u53d6\u5c5e\u6027\u3002\u5bf9\u4e8e\u6b64\u793a\u4f8b\uff0c\u6211\u4eec\u5c06\u4e86\u89e3\u5982\u4f55\u4ece\u7f51\u7ad9\u5c4f\u5e55\u622a\u56fe\u4e2d\u63d0\u53d6\u6709\u7528\u7684\u5c5e\u6027<\/p>\n

1\u3001\u5b9a\u4e49\u7b7e\u540d<\/h2>\n

\u5b9a\u4e49DSPy\u7b7e\u540d\u3002\u6ce8\u610f dspy.Image<\/code> \u8f93\u5165\u5b57\u6bb5\uff1a<\/p>\n

import dspy\nclass WebsiteDataExtractionSignature(dspy.Signature):\n    \"\"\"Website data extraction\"\"\"\n    website_screenshot: dspy.Image = dspy.InputField(\n        desc=\"A screenshot of the website\"\n    )\n    hero_text: str = dspy.OutputField(\n        desc=\"The hero text of the website\"\n    )\n    website_description: str = dspy.OutputField(\n        desc=\"A description of the website\"\n    )\n    call_to_action: str = dspy.OutputField(\n        desc=\"The call to action of the website\"\n    )\n    color_palette: list[str] = dspy.OutputField(\n        desc=\"The color palette of the website\"\n    )\n    font_palette: list[str] = dspy.OutputField(\n        desc=\"The font palette of the website\"\n    )<\/code><\/pre>\n
2\u3001\u5b9a\u4e49\u6a21\u5757<\/h2>\n
\u63a5\u4e0b\u6765\u4f7f\u7528 ChainOfThought<\/code> \u4f18\u5316\u5668\u548c\u4e0a\u4e00\u6b65\u4e2d\u7684\u7b7e\u540d\u5b9a\u4e49\u4e00\u4e2a\u7b80\u5355\u7684\u7a0b\u5e8f\uff1a<\/p>\n
class WebsiteDataExtraction(dspy.Module):\n    \"\"\"Module for extracting structured data from website screenshots.\"\"\"\n    def __init__(self):\n        self.website_data_extraction = dspy.ChainOfThought(\n            WebsiteDataExtractionSignature\n        )\n        \n    # pylint: disable=missing-function-docstring\n    def forward(self, website_screenshot: str):\n        website_data = self.website_data_extraction(website_screenshot)\n        return website_data<\/code><\/pre>\n
3\u3001\u6700\u7ec8\u4ee3\u7801<\/h2>\n
\u6700\u540e\uff0c\u7f16\u5199\u4e00\u4e2a\u51fd\u6570\u6765\u8bfb\u53d6\u56fe\u50cf\u5e76\u901a\u8fc7\u8c03\u7528\u4e0a\u4e00\u6b65\u4e2d\u7684\u7a0b\u5e8f\u6765\u63d0\u53d6\u5c5e\u6027\uff1a<\/p>\n
def extract_website_data(website_screenshot_path: str):\n    \"\"\"Extract data from a website screenshot.\n    \n    Args:\n        website_screenshot_path (str): Path to the website screenshot image\n    \n    Returns:\n        dict: Extracted website data\n    \"\"\"\n    # Load the image\n    with open(website_screenshot_path, \"rb\") as image_file:\n        base64_data = base64.b64encode(image_file.read()).decode('utf-8').replace('\\n', '')\n        image_data_uri = f\"data:image\/png;base64,{base64_data}\"\n    website_data_extraction = WebsiteDataExtraction()\n    website_data = website_data_extraction(image_data_uri)\n    return website_data\n\nif __name__ == \"__main__\":\n    dspy_lm = dspy.LM(model=\"openai\/gpt-4o-mini\")\n    dspy.config( lm=dspy_lm)\n    result = extract_website_data(\n        \"src\/vision_lm\/data\/langtrace-screenshot.png\"\n    )\n    print(result)<\/code><\/pre>\n
4\u3001\u53ef\u89c2\u5bdf\u6027<\/h2>\n
\u5c31\u662f\u8fd9\u6837\uff01\u5982\u679c\u60a8\u7684\u5f00\u53d1\u9700\u8981\u53ef\u89c2\u5bdf\u6027\uff0c\u53ea\u9700\u6dfb\u52a0 langtrace.init()<\/code> \u5373\u53ef\u4ece\u8ddf\u8e2a\u4e2d\u83b7\u5f97\u66f4\u6df1\u5165\u7684\u89c1\u89e3\u3002<\/p>\n
5\u3001\u6e90\u4ee3\u7801<\/h2>\n
\u4f60\u53ef\u4ee5\u5728\u6b64\u5904\u627e\u5230\u6b64\u793a\u4f8b\u7684\u3002<\/p>\n
\n","protected":false},"excerpt":{"rendered":"
DSPy \u6700\u8fd1\u5728\u6d4b\u8bd5\u7248\u4e2d\u589e\u52a0\u4e86\u5bf9 VLM \u7684\u652f\u6301\u3002\u672c\u6587\u4ecb\u7ecd\u4f7f\u7528 DSPy \u4ece\u56fe\u50cf\u4e2d\u63d0\u53d6\u5c5e\u6027\u3002\u5bf9\u4e8e\u6b64\u793a\u4f8b\uff0c\u6211\u4eec\u5c06\u4e86\u89e3\u5982\u4f55\u4ece\u7f51\u7ad9\u5c4f\u5e55\u622a\u56fe\u4e2d\u63d0\u53d6\u6709\u7528\u7684\u5c5e\u6027 1\u3001\u5b9a\u4e49\u7b7e\u540d \u5b9a\u4e49DSPy\u7b7e\u540d\u3002\u6ce8\u610f dspy.Image \u8f93\u5165\u5b57\u6bb5\uff1a import dspy class WebsiteDataExtractionSignature(dspy.Signature): “””Website data extraction””” website_screenshot: dspy.Image = dspy.InputField( desc=”A screenshot of the website” ) hero_text: str = dspy.OutputField( desc=”The hero text of the website” ) website_description: str = dspy.OutputField( desc=”A description of the website” ) call_to_action: str = dspy.OutputField( desc=”The call to […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[13],"tags":[],"class_list":["post-53738","post","type-post","status-publish","format-standard","hentry","category-ai"],"_links":{"self":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts\/53738","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/comments?post=53738"}],"version-history":[{"count":0,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/posts\/53738\/revisions"}],"wp:attachment":[{"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/media?parent=53738"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/categories?post=53738"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/fwq.ai\/blog\/wp-json\/wp\/v2\/tags?post=53738"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}