Skip to content

Commit

Permalink
Merge pull request #1533 from stanfordnlp/improve_adapter_raw_demos
Browse files Browse the repository at this point in the history 
Add deprecation warning for old clients on first use (per client)
  • Loading branch information
@okhat
okhat authored Sep 23, 2024
2 parents d03901e + e902ad4 commit 6ce999d
Show file tree
Hide file tree
Showing 2 changed files with 167 additions and 32 deletions.
21 changes: 18 additions & 3 deletions dspy/predict/predict.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@
from dspy.primitives.prediction import Prediction
from dspy.signatures.signature import ensure_signature, signature_to_template

import logging
from functools import lru_cache

@lru_cache(maxsize=None)
def warn_once(msg: str):
logging.warning(msg)


class Predict(Module, Parameter):
def __init__(self, signature, **config):
Expand Down Expand Up @@ -122,10 +129,18 @@ def forward(self, **kwargs):
import dspy
if isinstance(lm, dspy.LM):
completions = v2_5_generate(lm, config, signature, demos, kwargs)
elif dsp.settings.experimental:
completions = new_generate(lm, signature, dsp.Example(demos=demos, **kwargs), **config)
else:
completions = old_generate(demos, signature, kwargs, config, self.lm, self.stage)
warn_once("\t*** In DSPy 2.5, all LM clients except `dspy.LM` are deprecated. ***\n"
f" \t\tYou are using the client {lm.__class__.__name__}, which will be removed in DSPy 2.6.\n"
" \t\tChanging the client is straightforward and will let you use new features (Adapters) that"
" improve the consistency of LM outputs, especially when using chat LMs. \n\n"
" \t\tLearn more about the changes and how to migrate at\n"
" \t\thttps://github.com/stanfordnlp/dspy/blob/main/examples/migration.ipynb")

if dsp.settings.experimental:
completions = new_generate(lm, signature, dsp.Example(demos=demos, **kwargs), **config)
else:
completions = old_generate(demos, signature, kwargs, config, self.lm, self.stage)

pred = Prediction.from_completions(completions, signature=signature)

Expand Down
178 changes: 149 additions & 29 deletions examples/migration.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,9 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/anaconda3/envs/jun2024_py310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"outputs": [],
"source": [
"import dspy\n",
"\n",
Expand Down Expand Up @@ -75,7 +66,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -106,7 +97,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -129,7 +120,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -161,9 +152,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"['2 + 2 equals 4.']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lm(\"What is 2+2?\", temperature=0.9)"
]
Expand All @@ -177,9 +179,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"['2 + 2 equals 4.']"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lm(messages=[{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"What is 2+2?\"}])"
Expand All @@ -194,9 +207,23 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"Prediction(\n",
" reasoning='To solve the equation 2 + 2, we simply add the two numbers together. The number 2 represents a quantity, and when we add another 2 to it, we are combining these quantities. Therefore, 2 + 2 equals 4.',\n",
" answer='4'\n",
")"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"module = dspy.ChainOfThought('question -> answer')\n",
"module(question=\"What is 2+2?\")"
Expand All @@ -211,9 +238,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['prompt', 'messages', 'kwargs', 'response', 'outputs', 'usage', 'cost'])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(lm.history) # e.g., 3 calls to the LM\n",
"\n",
Expand All @@ -231,9 +269,23 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"Prediction(\n",
" reasoning='The first claim states that \"Python was released in 1991,\" which is true. Python was indeed first released by Guido van Rossum in February 1991. The second claim states that \"Python is a compiled language.\" This is false; Python is primarily an interpreted language, although it can be compiled to bytecode which is then interpreted by the Python virtual machine. Therefore, the first claim is true, and the second claim is false.',\n",
" verdict=[True, False]\n",
")"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fact_checking = dspy.ChainOfThought('claims -> verdict: list[bool]')\n",
"fact_checking(claims=[\"Python was released in 1991.\", \"Python is a compiled language.\"])"
Expand All @@ -248,9 +300,69 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"\n",
"\n",
"\u001b[31mSystem message:\u001b[0m\n",
"\n",
"Your input fields are:\n",
"1. `claims` (str)\n",
"\n",
"Your output fields are:\n",
"1. `reasoning` (str)\n",
"2. `verdict` (list[bool])\n",
"\n",
"All interactions will be structured in the following way, with the appropriate values filled in.\n",
"\n",
"[[ ## claims ## ]]\n",
"{claims}\n",
"\n",
"[[ ## reasoning ## ]]\n",
"{reasoning}\n",
"\n",
"[[ ## verdict ## ]]\n",
"{verdict}\n",
"\n",
"[[ ## completed ## ]]\n",
"\n",
"In adhering to this structure, your objective is: \n",
" Given the fields `claims`, produce the fields `verdict`.\n",
"\n",
"\n",
"\u001b[31mUser message:\u001b[0m\n",
"\n",
"[[ ## claims ## ]]\n",
"[1] «Python was released in 1991.»\n",
"[2] «Python is a compiled language.»\n",
"\n",
"Respond with the corresponding output fields, starting with the field `reasoning`, then `verdict`, and then ending with the marker for `completed`.\n",
"\n",
"\n",
"\u001b[31mResponse:\u001b[0m\n",
"\n",
"\u001b[32m[[ ## reasoning ## ]]\n",
"The first claim states that \"Python was released in 1991,\" which is true. Python was indeed first released by Guido van Rossum in February 1991. The second claim states that \"Python is a compiled language.\" This is false; Python is primarily an interpreted language, although it can be compiled to bytecode which is then interpreted by the Python virtual machine. Therefore, the first claim is true, and the second claim is false.\n",
"\n",
"[[ ## verdict ## ]]\n",
"[True, False]\n",
"\n",
"[[ ## completed ## ]]\u001b[0m\n",
"\n",
"\n",
"\n",
"\n",
"\n"
]
}
],
"source": [
"dspy.inspect_history()"
]
Expand All @@ -266,9 +378,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"When parsing the output: Expected dict_keys(['reasoning', 'output']) but got dict_keys(['reasoning'])\n"
]
}
],
"source": [
"# a module with insufficient tokens to format the output\n",
"bad_module = dspy.ChainOfThought('input -> output: list[str]', max_tokens=5)\n",
Expand Down Expand Up @@ -302,7 +422,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
Expand Down

0 comments on commit 6ce999d

Please sign in to comment.