[tcgc] sync with current typespec-python in main (#4477)

Co-authored-by: iscai-msft <isabellavcai@gmail.com>
Co-authored-by: Chenjie Shi <tadelesh.shi@live.cn>
Co-authored-by: Yuchao Yan <yuchaoyan@microsoft.com>

packages/http-client-python/emitter/src/code-model.ts

@@ -101,7 +101,7 @@ function emitMethodParameter<TServiceOperation extends SdkServiceOperation>(
 ): Record<string, any>[] {
   if (parameter.kind === "endpoint") {
     if (parameter.type.kind === "union") {
-      for (const endpointVal of parameter.type.values) {
+      for (const endpointVal of parameter.type.variantTypes) {
         return emitEndpointType(context, endpointVal as SdkEndpointType);
       }
     } else {
@@ -210,7 +210,7 @@ function emitClient<TServiceOperation extends SdkServiceOperation>(
   const operationGroups = emitOperationGroups(context, client, client, "");
   let url: string | undefined;
   if (endpointParameter?.type.kind === "union") {
-    url = (endpointParameter.type.values[0] as SdkEndpointType).serverUrl;
+    url = (endpointParameter.type.variantTypes[0] as SdkEndpointType).serverUrl;
   } else {
     url = endpointParameter?.type.serverUrl;
   }

packages/http-client-python/emitter/src/emitter.ts

@@ -79,7 +79,7 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
   const outputDir = context.emitterOutputDir;
   const yamlMap = emitCodeModel(sdkContext);
   addDefaultOptions(sdkContext);
-  const yamlPath = await saveCodeModelAsYaml("typespec-python-yaml-map", yamlMap);
+  const yamlPath = await saveCodeModelAsYaml("python-yaml-path", yamlMap);
   let venvPath = path.join(root, "venv");
   if (fs.existsSync(path.join(venvPath, "bin"))) {
     venvPath = path.join(venvPath, "bin", "python");

packages/http-client-python/emitter/src/http.ts

@@ -101,7 +101,7 @@ function addPagingInformation(
   method: SdkPagingServiceMethod<SdkHttpOperation> | SdkLroPagingServiceMethod<SdkHttpOperation>,
   operationGroupName: string,
 ) {
-  for (const response of method.operation.responses.values()) {
+  for (const response of method.operation.responses) {
     if (response.type) {
       getType(context, response.type)["usage"] = UsageFlags.None;
     }
@@ -164,11 +164,11 @@ function emitHttpOperation(
 ): Record<string, any> {
   const responses: Record<string, any>[] = [];
   const exceptions: Record<string, any>[] = [];
-  for (const [statusCodes, response] of operation.responses) {
-    responses.push(emitHttpResponse(context, statusCodes, response, method)!);
+  for (const response of operation.responses) {
+    responses.push(emitHttpResponse(context, response.statusCodes, response, method)!);
   }
-  for (const [statusCodes, exception] of operation.exceptions) {
-    exceptions.push(emitHttpResponse(context, statusCodes, exception, undefined, true)!);
+  for (const exception of operation.exceptions) {
+    exceptions.push(emitHttpResponse(context, exception.statusCodes, exception, undefined, true)!);
   }
   const result = {
     url: operation.path,

packages/http-client-python/emitter/src/lib.ts

@@ -48,7 +48,7 @@ const EmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {
 };
 
 const libDef = {
-  name: "@azure-tools/typespec-python",
+  name: "@typespec/http-client-python",
   diagnostics: {},
   emitter: {
     options: EmitterOptionsSchema as JSONSchemaType<PythonEmitterOptions>,

packages/http-client-python/emitter/src/types.ts

@@ -112,7 +112,7 @@ export function getType<TServiceOperation extends SdkServiceOperation>(
     case "string":
     case "url":
       return emitBuiltInType(type);
-    case "any":
+    case "unknown":
       return KnownTypes.any;
     case "nullable":
       return getType(context, type.type);
@@ -227,6 +227,7 @@ function emitProperty<TServiceOperation extends SdkServiceOperation>(
     if (body) {
       // for `temperature: HttpPart<{@body body: float64, @header contentType: "text/plain"}>`, the real type is float64
       sourceType = body.type;
+      addDisableGenerationMap(property.type);
     }
   }
   if (isMultipartFileInput) {
@@ -259,6 +260,13 @@ function emitModel<TServiceOperation extends SdkServiceOperation>(
     return typesMap.get(type)!;
   }
   if (type.crossLanguageDefinitionId === "Azure.Core.Foundations.Error") {
+    return {
+      type: "sdkcore",
+      name: "ODataV4Format",
+      submodule: "exceptions",
+    };
+  }
+  if (type.crossLanguageDefinitionId === "Azure.Core.Foundations.ErrorResponse") {
     return {
       type: "sdkcore",
       name: "HttpResponseError",
@@ -459,7 +467,7 @@ function emitUnion<TServiceOperation extends SdkServiceOperation>(
     description: type.isGeneratedName ? "" : `Type of ${type.name}`,
     internal: true,
     type: "combined",
-    types: type.values.map((x) => getType(context, x)),
+    types: type.variantTypes.map((x) => getType(context, x)),
     xmlMetadata: {},
   });
 }

packages/http-client-python/eng/scripts/Test-Packages.ps1

@@ -52,7 +52,7 @@ try {
             & pip list
             # Run tox
             Write-Host 'Running tests'
-            & npm run test
+            & npm run ci
             Write-Host 'tox tests passed'
         } 
         catch {

packages/http-client-python/eng/scripts/ci/run-tests.ts

@@ -6,8 +6,6 @@ import { dirname, join } from "path";
 import { fileURLToPath } from "url";
 import { parseArgs } from "util";
 
-const validCommands = ["ci", "lint", "mypy", "pyright", "apiview"];
-
 const root = join(dirname(fileURLToPath(import.meta.url)), "../../../");
 
 const argv = parseArgs({
@@ -24,10 +22,9 @@ const foldersToProcess = argv.values.flavor
   ? [argv.values.flavor]
   : argv.values.validFolders || ["azure", "unbranded"];
 
-const commandToRun = argv.values.command || "all";
+const commandToRun = argv.values.command || "ci";
 
 function getCommand(command: string, flavor: string, name?: string): string {
-  if (!validCommands.includes(command)) throw new Error(`Unknown command '${command}'.`);
   let retval: string;
   if (platform() === "win32") {
     retval = `set FOLDER=${flavor} && ${venvPath} -m tox -c ./test/${flavor}/tox.ini -e ${command}`;
@@ -44,8 +41,10 @@ function getCommand(command: string, flavor: string, name?: string): string {
 function sectionExistsInToxIni(command: string, flavor: string): boolean {
   const toxIniPath = join(root, `test/${flavor}/tox.ini`);
   const toxIniContent = readFileSync(toxIniPath, "utf-8");
-  const sectionHeader = `[testenv:${command}]`;
-  return toxIniContent.includes(sectionHeader);
+  return command
+    .split(",")
+    .map((c) => `[testenv:${c}]`)
+    .every((section) => toxIniContent.includes(section));
 }
 
 function myExecSync(command: string, flavor: string, name?: string): void {
@@ -65,23 +64,9 @@ if (fs.existsSync(join(venvPath, "bin"))) {
   throw new Error("Virtual environment doesn't exist.");
 }
 
-// Install dependencies from dev_requirements.txt
-const devRequirementsPath = join(root, "generator", "dev_requirements.txt");
-if (fs.existsSync(devRequirementsPath)) {
-  console.log("Installing dependencies from dev_requirements.txt...");
-  execSync(`${venvPath} -m pip install -r ${devRequirementsPath}`, { stdio: "inherit" });
-} else {
-  throw new Error("dev_requirements.txt doesn't exist.");
-}
-
 foldersToProcess.forEach((flavor) => {
   try {
-    if (commandToRun === "all") {
-      for (const key of validCommands) {
-        console.log(`Running ${key} for flavor ${flavor}...`);
-        myExecSync(key, flavor, argv.values.name);
-      }
-    } else if (getCommand(commandToRun, flavor, argv.values.name)) {
+    if (getCommand(commandToRun, flavor, argv.values.name)) {
       console.log(`Running ${commandToRun} for flavor ${flavor}...`);
       myExecSync(commandToRun, flavor, argv.values.name);
     } else {

packages/http-client-python/generator/pygen/codegen/models/imports.py

@@ -182,7 +182,7 @@ def define_mutable_mapping_type(self) -> None:
         self.add_mutable_mapping_import()
         self.define_mypy_type(
             "JSON",
-            "MutableMapping[str, Any]",
+            "MutableMapping[str, Any] # pylint: disable=unsubscriptable-object",
         )
         self.add_submodule_import("typing", "Any", ImportType.STDLIB)
 

packages/http-client-python/generator/pygen/codegen/models/operation.py

@@ -142,6 +142,11 @@ def pylint_disable(self, async_mode: bool) -> str:
         if not async_mode and not self.is_overload and self.response_type_annotation(async_mode=False) == "None":
             # doesn't matter if it's async or not
             retval = add_to_pylint_disable(retval, "inconsistent-return-statements")
+        try:
+            if any(is_internal(r.type) for r in self.responses) or is_internal(self.parameters.body_parameter.type):
+                retval = add_to_pylint_disable(retval, "protected-access")
+        except ValueError:
+            pass
         if len(self.name) > NAME_LENGTH_LIMIT:
             retval = add_to_pylint_disable(retval, "name-too-long")
         return retval

packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py

@@ -266,8 +266,8 @@ def description_and_summary(self, builder: BuilderType) -> List[str]:
         return description_list
 
     @staticmethod
-    def line_too_long(docs: List[str]) -> bool:
-        return any(len(line) > 120 for line in docs)
+    def line_too_long(docs: List[str], indentation: int = 0) -> bool:
+        return any(len(line) > (120 - indentation) for line in docs)
 
     def example_template(self, builder: BuilderType) -> List[str]:
         template = []
@@ -599,7 +599,7 @@ def _api_version_validation(self, builder: OperationType) -> str:
             retval.append(f"    params_added_on={dict(params_added_on)},")
         if retval:
             retval_str = "\n".join(retval)
-            return f"@api_version_validation(\n{retval_str}\n)"
+            return f"@api_version_validation(\n{retval_str}\n){builder.pylint_disable(self.async_mode)}"
         return ""
 
     def pop_kwargs_from_signature(self, builder: OperationType) -> List[str]:

packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py

@@ -278,10 +278,13 @@ def declare_property(prop: Property) -> str:
             else ""
         )
         generated_code = f'{prop.client_name}: {prop.type_annotation()} = {field}({", ".join(args)})'
-        decl = f"{generated_code}{type_ignore}"
-        if len(decl) + 4 > 120:
-            decl += "  # pylint: disable=line-too-long"
-        return decl
+        # there is 4 spaces indentation so original line length limit 120 - 4 = 116
+        pylint_disable = (
+            " # pylint: disable=line-too-long"
+            if len(generated_code) <= 116 < (len(generated_code) + len(type_ignore))
+            else ""
+        )
+        return f"{generated_code}{type_ignore}{pylint_disable}"
 
     def initialize_properties(self, model: ModelType) -> List[str]:
         init_args = []

packages/http-client-python/generator/pygen/codegen/serializers/test_serializer.py

@@ -200,12 +200,12 @@ def import_test(self) -> FileImportSerializer:
         test_name = TestName(self.code_model, self.client.name, is_async=self.is_async)
         async_suffix = "_async" if self.is_async else ""
         imports.add_submodule_import(
-            ("devtools_testutils" if self.code_model.options["azure_arm"] else "testpreparer" + async_suffix),
+            "devtools_testutils" if self.code_model.options["azure_arm"] else "testpreparer" + async_suffix,
             test_name.base_test_class_name,
             ImportType.LOCAL,
         )
         imports.add_submodule_import(
-            ("devtools_testutils" if self.code_model.options["azure_arm"] else "testpreparer"),
+            "devtools_testutils" if self.code_model.options["azure_arm"] else "testpreparer",
             test_name.preparer_name,
             ImportType.LOCAL,
         )

packages/http-client-python/generator/pygen/codegen/templates/conftest.py.jinja2

@@ -7,7 +7,7 @@ from devtools_testutils import test_proxy, add_general_regex_sanitizer, add_body
 
 load_dotenv()
 
-# avoid record sensitive identity information in recordings
+# For security, please avoid record sensitive identity information in recordings
 @pytest.fixture(scope="session", autouse=True)
 def add_sanitizers(test_proxy):
     {% for test_name in test_names %}

packages/http-client-python/generator/pygen/codegen/templates/model_base.py.jinja2

@@ -4,7 +4,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-# pylint: disable=protected-access, arguments-differ, signature-differs, broad-except
+# pylint: disable=protected-access, arguments-differ, signature-differs, broad-except, too-many-lines
 
 import copy
 import calendar
@@ -346,7 +346,7 @@ def _get_model(module_name: str, model_name: str):
 _UNSET = object()
 
 
-class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=unsubscriptable-object
+class _MyMutableMapping(MutableMapping[str, typing.Any]):  # pylint: disable=unsubscriptable-object
     def __init__(self, data: typing.Dict[str, typing.Any]) -> None:
         self._data = data
 
@@ -573,7 +573,7 @@ class Model(_MyMutableMapping):
     def copy(self) -> "Model":
         return Model(self.__dict__)
 
-    def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self:
+    def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self:  # pylint: disable=unused-argument
         if f"{cls.__module__}.{cls.__qualname__}" not in cls._calculated:
             # we know the last nine classes in mro are going to be 'Model', '_MyMutableMapping', 'MutableMapping',
             # 'Mapping', 'Collection', 'Sized', 'Iterable', 'Container' and 'object'
@@ -584,8 +584,8 @@ class Model(_MyMutableMapping):
             annotations = {
                 k: v
                 for mro_class in mros
-                if hasattr(mro_class, "__annotations__")
-                for k, v in mro_class.__annotations__.items()
+                if hasattr(mro_class, "__annotations__")  # pylint: disable=no-member
+                for k, v in mro_class.__annotations__.items()  # pylint: disable=no-member
             }
             for attr, rf in attr_to_rest_field.items():
                 rf._module = cls.__module__
@@ -596,12 +596,12 @@ class Model(_MyMutableMapping):
             cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items())
             cls._calculated.add(f"{cls.__module__}.{cls.__qualname__}")
 
-        return super().__new__(cls) # pylint: disable=no-value-for-parameter
+        return super().__new__(cls)  # pylint: disable=no-value-for-parameter
 
     def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None:
         for base in cls.__bases__:
-            if hasattr(base, "__mapping__"):
-                base.__mapping__[discriminator or cls.__name__] = cls  # type: ignore
+            if hasattr(base, "__mapping__"):  # pylint: disable=no-member
+                base.__mapping__[discriminator or cls.__name__] = cls  # type: ignore  # pylint: disable=no-member
 
     @classmethod
     def _get_discriminator(cls, exist_discriminators) -> typing.Optional["_RestField"]:
@@ -612,7 +612,7 @@ class Model(_MyMutableMapping):
 
     @classmethod
     def _deserialize(cls, data, exist_discriminators):
-        if not hasattr(cls, "__mapping__"):
+        if not hasattr(cls, "__mapping__"):  # pylint: disable=no-member
             return cls(data)
         discriminator = cls._get_discriminator(exist_discriminators)
         if discriminator is None:

packages/http-client-python/generator/pygen/codegen/templates/operation_tools.jinja2

@@ -2,18 +2,25 @@
 
 {% macro description(builder, serializer) %}
 {% set example_template = serializer.example_template(builder) %}
+{% set param_description_and_response_docstring = serializer.param_description_and_response_docstring(builder) %}
+{% set ns = namespace(line_too_long=false) %}
+{% for item in param_description_and_response_docstring %}
+    {% if item and serializer.line_too_long(wrap_string(item, wrapstring='\n ').split('\n'), 8) %}
+        {% set ns.line_too_long = true %}
+    {% endif %}
+{% endfor %}
     {% for description in serializer.description_and_summary(builder) %}
         {% if description %}
 {% set description = wrap_string(description, wrapstring='\n') %}
-            {% if serializer.line_too_long(example_template) and loop.first %}
+            {% if (serializer.line_too_long(example_template) or ns.line_too_long) and loop.first %}
 # pylint: disable=line-too-long
             {% endif %}
 {{ '"""' + description if loop.first else description }}
         {% else %}
 
         {% endif %}
     {% endfor %}
-    {% for description in serializer.param_description_and_response_docstring(builder) %}
+    {% for description in param_description_and_response_docstring %}
         {% if description %}
 {{ wrap_string(description, wrapstring='\n ') }}
         {% else %}