NAS-130523 / 25.04 / Prototype implementation for Container support using Incus (WIP)

src/freenas/debian/preinst

@@ -3,6 +3,8 @@
 mkdir -p /var/trash
 for file in \
     /etc/nsswitch.conf \
+    /etc/subgid \
+    /etc/subuid \
     /usr/lib/netdata/conf.d/python.d.conf \
     /usr/lib/netdata/conf.d/charts.d.conf \
     /lib/systemd/system/smartmontools.service \

src/freenas/etc/subgid

@@ -0,0 +1 @@
+0:1000000:1000000000

src/freenas/etc/subuid

@@ -0,0 +1 @@
+0:1000000:1000000000

src/middlewared/debian/control

@@ -66,6 +66,7 @@ Build-Depends: alembic,
                python3-systemd,
                python3-truenas-api-client,
                python3-truenas-verify,
+               python3-usb,
                python3-websocket,
                python3-zeroconf,
                python3-zettarepl,

src/middlewared/middlewared/alembic/versions/25.04/2024-09-20_14-03_virt_global.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: dd6e581235b2
+Revises: 504a7bd32680
+Create Date: 2024-08-02 14:03:41.855489+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'dd6e581235b2'
+down_revision = '6dedf12c1035'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('virt_global',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('pool', sa.String(length=120), nullable=True),
+    sa.Column('bridge', sa.String(length=120), nullable=True),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_virt_global')),
+    sqlite_autoincrement=True
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('virt_global')
+    # ### end Alembic commands ###

src/middlewared/middlewared/api/v25_04_0/__init__.py

@@ -12,3 +12,4 @@
 from .system_lifecycle import *  # noqa
 from .user import *  # noqa
 from .vendor import *  # noqa
+from .virt import *  #noqa

src/middlewared/middlewared/api/v25_04_0/virt.py

@@ -0,0 +1,265 @@
+from typing import Literal, List, Union, Optional, TypeAlias
+from typing_extensions import Annotated
+
+from pydantic import Field, StringConstraints
+
+from middlewared.api.base import (
+    BaseModel, ForUpdateMetaclass, NonEmptyString,
+    single_argument_args, single_argument_result,
+)
+
+
+class VirtGlobalEntry(BaseModel):
+    id: int
+    pool: str | None = None
+    dataset: str | None = None
+    bridge: str | None = None
+    state: str | None = None
+
+
+@single_argument_args('virt_global_update')
+class VirtGlobalUpdateArgs(BaseModel):
+    pool: str | None = None
+    bridge: str | None = None
+
+
+class VirtGlobalUpdateResult(BaseModel):
+    result: VirtGlobalEntry
+
+
+class VirtGlobalBridgeChoicesArgs(BaseModel):
+    pass
+
+
+class VirtGlobalBridgeChoicesResult(BaseModel):
+    result: dict
+
+
+class VirtGlobalPoolChoicesArgs(BaseModel):
+    pass
+
+
+class VirtGlobalPoolChoicesResult(BaseModel):
+    result: dict
+
+
+class VirtGlobalGetNetworkArgs(BaseModel):
+    name: NonEmptyString
+
+
+@single_argument_result
+class VirtGlobalGetNetworkResult(BaseModel):
+    type: Literal['BRIDGE']
+    managed: bool
+    ipv4_address: NonEmptyString
+    ipv4_nat: bool
+    ipv6_address: NonEmptyString
+    ipv6_nat: bool
+
+
+REMOTE_CHOICES: TypeAlias = Optional[Literal['LINUX_CONTAINERS']]
+
+
+@single_argument_args('virt_instances_image_choices')
+class VirtInstancesImageChoicesArgs(BaseModel):
+    remote: REMOTE_CHOICES = None
+
+
+class ImageChoiceItem(BaseModel):
+    label: str
+    os: str
+    release: str
+    arch: str
+    variant: int
+
+
+class VirtInstancesImageChoicesResult(BaseModel):
+    result: dict[str, ImageChoiceItem]
+
+
+class Device(BaseModel):
+    name: Optional[NonEmptyString] = None
+    dev_type: Literal['USB', 'TPM', 'DISK', 'GPU', 'NIC', 'PROXY']
+    readonly: bool = False
+
+
+class Disk(Device):
+    source: Optional[str] = None
+    destination: Optional[str] = None
+
+
+class NIC(Device):
+    network: NonEmptyString
+
+
+class USB(Device):
+    bus: Optional[int] = None
+    dev: Optional[int] = None
+    product_id: Optional[str] = None
+    vendor_id: Optional[str] = None
+
+
+Proto: TypeAlias = Literal['UDP', 'TCP']
+
+
+class Proxy(Device):
+    source_proto: Proto
+    source_port: int
+    dest_proto: Proto
+    dest_port: int
+
+
+class TPM(Device):
+    path: Optional[str] = None
+    pathrm: Optional[str] = None
+
+
+GPUType: TypeAlias = Literal['PHYSICAL', 'MDEV', 'MIG', 'SRIOV']
+
+
+class GPU(Device):
+    gpu_type: GPUType
+    id: str | None = None
+    gid: Optional[int] = None
+    uid: Optional[int] = None
+    mode: Optional[int] = None
+    mdev: Optional[NonEmptyString] = None
+    mig_uuid: Optional[NonEmptyString] = None
+    pci: Optional[NonEmptyString] = None
+    productid: Optional[NonEmptyString] = None
+    vendorid: Optional[NonEmptyString] = None
+
+
+Devices: TypeAlias = List[Union[Disk, GPU, Proxy, TPM, USB]]
+
+
+class VirtInstanceAlias(BaseModel):
+    type: Literal['INET', 'INET6']
+    address: NonEmptyString
+    netmask: int
+
+
+InstanceType: TypeAlias = Literal['CONTAINER', 'VM']
+
+
+class VirtInstanceEntry(BaseModel):
+    id: str
+    name: Annotated[NonEmptyString, StringConstraints(max_length=200)]
+    type: InstanceType = 'CONTAINER'
+    status: Literal['RUNNING', 'STOPPED']
+    cpu: str | None
+    memory: int
+    autostart: bool
+    environment: dict[str, str]
+    aliases: List[VirtInstanceAlias]
+    raw: dict
+
+
+@single_argument_args('virt_instance_create')
+class VirtInstancesCreateArgs(BaseModel):
+    name: Annotated[NonEmptyString, StringConstraints(max_length=200)]
+    image: Annotated[NonEmptyString, StringConstraints(max_length=200)]
+    remote: REMOTE_CHOICES = None
+    instance_type: InstanceType = 'CONTAINER'
+    environment: dict | None = None
+    autostart: bool | None = None
+    cpu: str | None = None
+    memory: int | None = None
+    devices: Devices = None
+
+
+class VirtInstancesCreateResult(BaseModel):
+    result: dict
+
+
+class VirtInstancesUpdate(BaseModel, metaclass=ForUpdateMetaclass):
+    environment: dict | None = None
+    autostart: bool | None = None
+    cpu: str | None = None
+    memory: int | None = None
+
+
+class VirtInstancesUpdateArgs(BaseModel):
+    id: str
+    virt_instance_update: VirtInstancesUpdate
+
+
+class VirtInstancesUpdateResult(BaseModel):
+    result: VirtInstanceEntry
+
+
+class VirtInstancesDeleteArgs(BaseModel):
+    id: str
+
+
+class VirtInstancesDeleteResult(BaseModel):
+    result: Literal[True]
+
+
+class VirtInstancesDeviceListArgs(BaseModel):
+    id: str
+
+
+class VirtInstancesDeviceListResult(BaseModel):
+    result: List[Devices]
+
+
+class VirtInstancesDeviceAddArgs(BaseModel):
+    id: str
+    device: Union[Disk, GPU, NIC, Proxy, TPM, USB] = Field(..., descriminator='dev_type')
+
+
+class VirtInstancesDeviceAddResult(BaseModel):
+    result: dict
+
+
+class VirtInstancesDeviceDeleteArgs(BaseModel):
+    id: str
+    name: str
+
+
+class VirtInstancesDeviceDeleteResult(BaseModel):
+    result: dict
+
+
+class VirtInstancesStateArgs(BaseModel):
+    id: str
+    action: Literal['START', 'STOP', 'RESTART']
+    force: bool = False
+
+
+class VirtInstancesStateResult(BaseModel):
+    result: bool
+
+
+class VirtDeviceUSBChoicesArgs(BaseModel):
+    pass
+
+
+class USBChoice(BaseModel):
+    vendor_id: str
+    product_id: str
+    bus: int
+    dev: int
+    product: str
+    manufacturer: str
+
+
+class VirtDeviceUSBChoicesResult(BaseModel):
+    result: dict[str, USBChoice]
+
+
+class VirtDeviceGPUChoicesArgs(BaseModel):
+    instance_type: InstanceType
+    gpu_type: GPUType
+
+
+class GPUChoice(BaseModel):
+    bus: int
+    slot: int
+    description: str
+    vendor: Optional[str] = None
+
+
+class VirtDeviceGPUChoicesResult(BaseModel):
+    result: dict[str, GPUChoice]

src/middlewared/middlewared/main.py

@@ -556,7 +556,7 @@ def __init__(self, middleware, ws, input_queue, loop, username, as_root, options
         super(ShellWorkerThread, self).__init__(daemon=True)
 
     def get_command(self, username, as_root, options):
-        allowed_options = ('vm_id', 'app_name')
+        allowed_options = ('vm_id', 'app_name', 'virt_instances_id')
         if all(options.get(k) for k in allowed_options):
             raise CallError(f'Only one option is supported from {", ".join(allowed_options)}')
 
@@ -575,6 +575,12 @@ def get_command(self, username, as_root, options):
                 '/usr/bin/docker', 'exec', '-it', options['container_id'], options.get('command', '/bin/bash'),
             ]
 
+            if not as_root:
+                command = ['/usr/bin/sudo', '-H', '-u', username] + command
+
+            return command, not as_root
+        elif options.get('virt_instances_id'):
+            command = ['/usr/bin/incus', 'exec', options['virt_instances_id'], options.get('command', '/bin/bash')]
             if not as_root:
                 command = ['/usr/bin/sudo', '-H', '-u', username] + command
 

src/middlewared/middlewared/plugins/interface/internal_ifaces.py

@@ -14,6 +14,7 @@ async def internal_interfaces(self):
         result = list(netif.INTERNAL_INTERFACES)
         result.extend(await self.middleware.call('failover.internal_interface.detect'))
         result.extend(await self.middleware.call('rdma.interface.internal_interfaces'))
+        result.extend(await self.middleware.call('virt.global.internal_interfaces'))
         if (await self.middleware.call('truenas.get_chassis_hardware')).startswith('TRUENAS-F'):
             # The eno1 interface needs to be masked on the f-series platform because
             # this interface is shared with the BMC. Details for why this is done

src/middlewared/middlewared/plugins/reporting/realtime_reporting/cgroup.py

@@ -0,0 +1,20 @@
+import collections
+import typing
+
+
+def get_cgroup_stats(netdata_metrics: dict, cgroups: typing.List[str]) -> dict[str, dict]:
+    data = collections.defaultdict(dict)
+    cgroup_keys = list(filter(lambda x: x.startswith('cgroup_'), netdata_metrics.keys()))
+
+    for cgroup in cgroups:
+        for i in filter(lambda x: x.startswith(f'cgroup_{cgroup}.'), cgroup_keys):
+            name = i.split('.', 1)[-1]
+            context = data[cgroup][name] = {}
+            metric = netdata_metrics[i]
+            unit = metric["units"].lower()
+            unit = unit.replace('/', '_')
+            for dimension, value in metric['dimensions'].items():
+                dimension = dimension.replace(' ', '_')
+                context[f'{name}_{dimension}_{unit}'] = value['value']
+
+    return data

src/middlewared/middlewared/plugins/service_/services/all.py

@@ -22,6 +22,7 @@
 from .pseudo.misc import (
     CronService,
     KmipService,
+    IncusService,
     LoaderService,
     HostnameService,
     HttpService,
@@ -64,6 +65,7 @@
     LibvirtGuestService,
     CronService,
     KmipService,
+    IncusService,
     LoaderService,
     HostnameService,
     HttpService,