Merge branch 'main' into feature/atkgen_custom_generator

garak/generators/test.py

@@ -5,6 +5,8 @@
 
 from typing import List
 
+import lorem
+
 from garak.generators.base import Generator
 
 
@@ -47,4 +49,17 @@ def _call_model(self, prompt: str, generations_this_call: int = 1) -> List[str]:
             )
 
 
-DEFAULT_CLASS = "Blank"
+class Lipsum(Generator):
+    """Lorem Ipsum generator, so we can get non-zero outputs that vary"""
+
+    supports_multiple_generations = False
+    generator_family_name = "Test"
+    name = "Lorem Ipsum"
+
+    def _call_model(
+        self, prompt: str, generations_this_call: int = 1
+    ) -> List[str | None]:
+        return [lorem.sentence() for i in range(generations_this_call)]
+
+
+DEFAULT_CLASS = "Lipsum"

pyproject.toml

@@ -67,6 +67,7 @@ dependencies = [
   "fschat>=0.2.36",
   "litellm>=1.33.8",
   "jsonpath-ng>=1.6.1",
+  "lorem==0.1.1"
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -28,6 +28,7 @@ deepl==1.17.0
 fschat>=0.2.36
 litellm>=1.33.8
 jsonpath-ng>=1.6.1
+lorem==0.1.1
 # tests
 pytest>=8.0
 requests-mock==1.12.1

tests/generators/test_test.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+import garak._plugins
+import garak.generators.base
+import garak.generators.test
+
+TEST_GENERATORS = [
+    a
+    for a, b in garak._plugins.enumerate_plugins("generators")
+    if b is True and a.startswith("generators.test")
+]
+
+
+@pytest.mark.parametrize("klassname", TEST_GENERATORS)
+def test_test_instantiate(klassname):
+    g = garak._plugins.load_plugin(klassname)
+    assert isinstance(g, garak.generators.base.Generator)
+
+
+@pytest.mark.parametrize("klassname", TEST_GENERATORS)
+def test_test_gen(klassname):
+    g = garak._plugins.load_plugin(klassname)
+    for generations in (1, 50):
+        out = g.generate("", generations_this_call=generations)
+        assert isinstance(out, list), ".generate() must return a list"
+        assert (
+            len(out) == generations
+        ), ".generate() must respect generations_per_call param"
+        for s in out:
+            assert (
+                isinstance(s, str) or s is None
+            ), "generate()'s returned list's items must be string or None"