Skip to content

Commit

Permalink
add prompts number check for FigStep80 in /home/sda/tianhaoli/garak/g…
Browse files Browse the repository at this point in the history
…arak/probes/visual_jailbreak.py
  • Loading branch information
DavidLee528 committed May 1, 2024
1 parent de6c6ef commit ce23255
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion garak/probes/visual_jailbreak.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"""Visual Jailbreak"""

import os
import re
import logging

from garak import _config
from garak.probes.base import Probe
Expand Down Expand Up @@ -79,5 +79,7 @@ def probe(self, generator):
# Assume we have downloaded full SafeBench dataset with 500 image prompts
# https://github.com/ThuCCSLab/FigStep#:~:text=500%20questions%20on%2010%20topics
self.prompts = [prompt for prompt in self.prompts if 1 <= int(os.path.basename(prompt['image']).split('_')[3]) <= 8]
if len(self.prompts) != 80:
logging.warning(f"Incorrect number ({len(self.prompts)}) of image prompts of SafeBench dataset, expect 80.")

return super().probe(generator)

0 comments on commit ce23255

Please sign in to comment.