fix error when model property is invalid (#3574)

fix #3504

When unsuitable types are specified in the model property, the invalid
code location is now reported instead of throwing an error.

When compiling the following tsp file, it is reported as follows
```tsp
interface TestInterface1 {
  get1(prop: TestInterface1): TestInterface1;
}
```

```
Diagnostics were reported during compilation:

/projects/typespec/packages/samples/scratch/main.tsp:2:8 - error @typespec/openapi3/invalid-model-property: 'Interface' cannot be specified as a model property.
> 2 |   get1(prop: TestInterface1): TestInterface1;
    |        ^^^^

Found 1 error.
```

---------

Co-authored-by: Timothee Guerin <tiguerin@microsoft.com>
Co-authored-by: Timothee Guerin <timothee.guerin@outlook.com>

.chronus/changes/main-2024-5-13-14-52-31.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/openapi3"
+---
+
+Emit diagnostic when an invalid type is used as a property instead of crashing.

packages/openapi3/src/lib.ts

@@ -245,6 +245,12 @@ export const libDef = {
         default: paramMessage`Status code range '${"start"} to '${"end"}' is not supported. OpenAPI 3.0 can only represent range 1XX, 2XX, 3XX, 4XX and 5XX. Example: \`@minValue(400) @maxValue(499)\` for 4XX.`,
       },
     },
+    "invalid-model-property": {
+      severity: "error",
+      messages: {
+        default: paramMessage`'${"type"}' cannot be specified as a model property.`,
+      },
+    },
     "unsupported-auth": {
       severity: "warning",
       messages: {

packages/openapi3/src/schema-emitter.ts

@@ -351,7 +351,14 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter<
     });
 
     if (refSchema.kind !== "code") {
-      throw new Error("Unexpected non-code result from emit reference");
+      reportDiagnostic(program, {
+        code: "invalid-model-property",
+        format: {
+          type: prop.type.kind,
+        },
+        target: prop,
+      });
+      return {};
     }
 
     const isRef = refSchema.value instanceof Placeholder || "$ref" in refSchema.value;