Merge remote-tracking branch 'origin/master' into NAS-130745

src/middlewared/middlewared/alembic/versions/25.04/2024-09-12_23-57_remove_nfs_v4_v3owner.py

@@ -0,0 +1,26 @@
+"""Remove deprecated v4_v3owner NFS configuration option
+
+
+Revision ID: 6dedf12c1035
+Revises: 7b618b9ca77d
+Create Date: 2024-09-12 23:57:43.814512+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6dedf12c1035'
+down_revision = '7b618b9ca77d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('services_nfs', schema=None) as batch_op:
+        batch_op.drop_column('nfs_srv_v4_v3owner')
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/alert/source/failover.py

@@ -55,9 +55,7 @@ class FailoverAlertSource(AlertSource):
     run_on_backup_node = False
 
     async def check(self):
-        if not await self.middleware.call('failover.licensed'):
-            return []
-        elif not await self.middleware.call('failover.internal_interfaces'):
+        if not await self.middleware.call('failover.internal_interfaces'):
             return [Alert(FailoverInterfaceNotFoundAlertClass)]
 
         try:

src/middlewared/middlewared/alert/source/failover_disks.py

@@ -31,8 +31,7 @@ class FailoverDisksAlertSource(AlertSource):
     run_on_backup_node = False
 
     async def check(self):
-        licensed = await self.middleware.call('failover.licensed')
-        if licensed and (md := await self.middleware.call('failover.mismatch_disks')):
+        if (md := await self.middleware.call('failover.mismatch_disks')):
             if md['missing_remote']:
                 return [Alert(
                     DisksAreNotPresentOnStandbyNodeAlertClass, {'serials': ', '.join(md['missing_remote'])}

src/middlewared/middlewared/alert/source/failover_interfaces.py

@@ -20,8 +20,7 @@ class FailoverCriticalAlertSource(AlertSource):
     run_on_backup_node = False
 
     async def check(self):
-        licensed = await self.middleware.call('failover.licensed')
-        if licensed and not await self.middleware.call('interface.query', [('failover_critical', '=', True)]):
+        if not await self.middleware.call('interface.query', [('failover_critical', '=', True)]):
             return [Alert(NoCriticalFailoverInterfaceFoundAlertClass)]
         else:
             return []

src/middlewared/middlewared/alert/source/failover_nics.py

@@ -31,8 +31,7 @@ class FailoverNetworkCardsAlertSource(AlertSource):
     run_on_backup_node = False
 
     async def check(self):
-        licensed = await self.middleware.call('failover.licensed')
-        if licensed and (interfaces := await self.middleware.call('failover.mismatch_nics')):
+        if (interfaces := await self.middleware.call('failover.mismatch_nics')):
             if interfaces['missing_remote']:
                 return [Alert(
                     NetworkCardsMismatchOnStandbyNodeAlertClass, {'interfaces': ', '.join(interfaces['missing_remote'])}

src/middlewared/middlewared/api/base/types/base.py

@@ -1,12 +1,14 @@
 from typing import Any, Generic, get_args, get_origin, TypeVar
 
-from pydantic import BeforeValidator, Field, GetCoreSchemaHandler, PlainSerializer
+from pydantic import AfterValidator, BeforeValidator, Field, GetCoreSchemaHandler, HttpUrl as _HttpUrl, PlainSerializer
 from pydantic_core import CoreSchema, core_schema, PydanticKnownError
 from typing_extensions import Annotated
 
 from middlewared.utils.lang import undefined
 
-__all__ = ["LongString", "NonEmptyString", "Private", "PRIVATE_VALUE"]
+__all__ = ["HttpUrl", "LongString", "NonEmptyString", "Private", "PRIVATE_VALUE"]
+
+HttpUrl = Annotated[_HttpUrl, AfterValidator(str)]
 
 
 class LongStringWrapper:

src/middlewared/middlewared/api/v25_04_0/__init__.py

@@ -4,6 +4,7 @@
 from .common import *  # noqa
 from .core import *  # noqa
 from .group import *  # noqa
+from .keychain import *  # noqa
 from .privilege import *  # noqa
 from .user import *  # noqa
 from .vendor import *  # noqa

src/middlewared/middlewared/api/v25_04_0/keychain.py

@@ -0,0 +1,164 @@
+from typing import Literal
+
+from pydantic import Field
+
+from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, HttpUrl, NonEmptyString,
+                                  Private, single_argument_args, single_argument_result)
+
+__all__ = ["KeychainCredentialEntry",
+           "KeychainCredentialCreateArgs", "KeychainCredentialCreateResult",
+           "KeychainCredentialUpdateArgs", "KeychainCredentialUpdateResult",
+           "KeychainCredentialDeleteArgs", "KeychainCredentialDeleteResult",
+           "KeychainCredentialUsedByArgs", "KeychainCredentialUsedByResult",
+           "KeychainCredentialGetOfTypeArgs", "KeychainCredentialGetOfTypeResult",
+           "KeychainCredentialGenerateSSHKeyPairArgs", "KeychainCredentialGenerateSSHKeyPairResult",
+           "KeychainCredentialRemoteSSHHostKeyScanArgs", "KeychainCredentialRemoteSSHHostKeyScanResult",
+           "KeychainCredentialRemoteSSHSemiautomaticSetupArgs", "KeychainCredentialRemoteSSHSemiautomaticSetupResult",
+           "KeychainCredentialSSHPairArgs", "KeychainCredentialSSHPairResult",
+           "KeychainCredentialSetupSSHConnectionArgs", "KeychainCredentialSetupSSHConnectionResult"]
+
+
+class KeychainCredentialEntry(BaseModel):
+    id: int
+    name: NonEmptyString
+    type: str
+    attributes: Private[dict]
+
+
+class KeychainCredentialCreate(KeychainCredentialEntry):
+    id: Excluded = excluded_field()
+
+
+class KeychainCredentialUpdate(KeychainCredentialCreate, metaclass=ForUpdateMetaclass):
+    type: Excluded = excluded_field()
+
+
+class KeychainCredentialCreateArgs(BaseModel):
+    keychain_credential_create: KeychainCredentialCreate
+
+
+class KeychainCredentialCreateResult(BaseModel):
+    result: KeychainCredentialEntry
+
+
+class KeychainCredentialUpdateArgs(BaseModel):
+    id: int
+    keychain_credential_update: KeychainCredentialUpdate
+
+
+class KeychainCredentialUpdateResult(BaseModel):
+    result: KeychainCredentialEntry
+
+
+class KeychainCredentialDeleteOptions(BaseModel):
+    cascade: bool = False
+
+
+class KeychainCredentialDeleteArgs(BaseModel):
+    id: int
+    options: KeychainCredentialDeleteOptions = Field(default=KeychainCredentialDeleteOptions())
+
+
+class KeychainCredentialDeleteResult(BaseModel):
+    result: None
+
+
+class KeychainCredentialUsedByArgs(BaseModel):
+    id: int
+
+
+class UsedKeychainCredential(BaseModel):
+    title: str
+    unbind_method: Literal["delete", "disable"]
+
+
+class KeychainCredentialUsedByResult(BaseModel):
+    result: list[UsedKeychainCredential]
+
+
+class KeychainCredentialGetOfTypeArgs(BaseModel):
+    id: int
+    type: str
+
+
+@single_argument_result
+class KeychainCredentialGetOfTypeResult(KeychainCredentialEntry):
+    pass
+
+
+class KeychainCredentialGenerateSSHKeyPairArgs(BaseModel):
+    pass
+
+
+@single_argument_result
+class KeychainCredentialGenerateSSHKeyPairResult(BaseModel):
+    private_key: str
+    public_key: str
+
+
+@single_argument_args("keychain_remote_ssh_host_key_scan")
+class KeychainCredentialRemoteSSHHostKeyScanArgs(BaseModel):
+    host: NonEmptyString
+    port: int = 22
+    connect_timeout: int = 10
+
+
+class KeychainCredentialRemoteSSHHostKeyScanResult(BaseModel):
+    result: str
+
+
+@single_argument_args("keychain_remote_ssh_semiautomatic_setup")
+class KeychainCredentialRemoteSSHSemiautomaticSetupArgs(BaseModel):
+    name: NonEmptyString
+    url: HttpUrl
+    verify_ssl: bool = True
+    token: Private[str | None] = None
+    admin_username: str = "root"
+    password: Private[str | None] = None
+    otp_token: Private[str | None] = None
+    username: str = "root"
+    private_key: Private[int]
+    connect_timeout: int = 10
+    sudo: bool = False
+
+
+class KeychainCredentialRemoteSSHSemiautomaticSetupResult(BaseModel):
+    result: KeychainCredentialEntry
+
+
+@single_argument_args("keychain_ssh_pair")
+class KeychainCredentialSSHPairArgs(BaseModel):
+    remote_hostname: NonEmptyString
+    username: str = "root"
+    public_key: NonEmptyString
+
+
+class KeychainCredentialSSHPairResult(BaseModel):
+    result: None
+
+
+class KeychainCredentialSetupSSHConnectionPrivateKey(BaseModel):
+    generate_key: bool = True
+    existing_key_id: int | None = None
+    name: NonEmptyString
+
+
+class KeychainCredentialSetupSSHConnectionSemiAutomaticSetup(
+    KeychainCredentialRemoteSSHSemiautomaticSetupArgs.model_fields["keychain_remote_ssh_semiautomatic_setup"].annotation
+):
+    name: Excluded = excluded_field()
+    private_key: Excluded = excluded_field()
+
+
+@single_argument_args("setup_ssh_connection")
+class KeychainCredentialSetupSSHConnectionArgs(BaseModel):
+    private_key: KeychainCredentialSetupSSHConnectionPrivateKey | None = None
+    connection_name: NonEmptyString
+    setup_type: Literal["SEMI-AUTOMATIC", "MANUAL"] = "MANUAL"
+    semi_automatic_setup: KeychainCredentialSetupSSHConnectionSemiAutomaticSetup | None = None
+    manual_setup: dict | None = None
+
+
+@single_argument_result
+class KeychainCredentialSetupSSHConnectionResult(KeychainCredentialEntry):
+    pass

src/middlewared/middlewared/etc_files/motd.mako

@@ -5,8 +5,9 @@
 
 	TrueNAS (c) 2009-${buildtime.year}, iXsystems, Inc.
 	All rights reserved.
-	TrueNAS code is released under the modified BSD license with some
-	files copyrighted by (c) iXsystems, Inc.
+	TrueNAS code is released under the LGPLv3 and GPLv3 licenses with some
+	source files copyrighted by (c) iXsystems, Inc. All other components
+	are released under their own respective licenses.
 
 	For more information, documentation, help or support, go here:
 	http://truenas.com

src/middlewared/middlewared/logger.py

@@ -39,6 +39,7 @@
 logging.getLogger('docker.auth').setLevel(logging.ERROR)
 logging.TRACE = 6
 
+APP_LIFECYCLE_LOGFILE = '/var/log/app_lifecycle.log'
 APP_MIGRATION_LOGFILE = '/var/log/app_migrations.log'
 DOCKER_IMAGE_LOGFILE = '/var/log/docker_image.log'
 FAILOVER_LOGFILE = '/var/log/failover.log'
@@ -83,6 +84,7 @@ def configure_logging(self, output_option: str):
         else:
             for name, filename, log_format in [
                 (None, LOGFILE, self.log_format),
+                ('app_lifecycle', APP_LIFECYCLE_LOGFILE, self.log_format),
                 ('app_migrations', APP_MIGRATION_LOGFILE, self.log_format),
                 ('docker_image', DOCKER_IMAGE_LOGFILE, self.log_format),
                 ('failover', FAILOVER_LOGFILE, self.log_format),

src/middlewared/middlewared/plugins/account.py

@@ -33,8 +33,6 @@
 from middlewared.plugins.idmap_.idmap_constants import (
     BASE_SYNTHETIC_DATASTORE_ID,
     IDType,
-    SID_LOCAL_USER_PREFIX,
-    SID_LOCAL_GROUP_PREFIX
 )
 from middlewared.plugins.idmap_ import idmap_winbind
 from middlewared.plugins.idmap_ import idmap_sss
@@ -1022,9 +1020,23 @@ def get_user_obj(self, data):
             user_obj['grouplist'] = None
 
         if data['sid_info']:
+            sid = None
             match user_obj['source']:
-                case 'LOCAL' | 'ACTIVEDIRECTORY':
-                    # winbind provides idmapping for local and AD users
+                case 'LOCAL':
+                    idmap_ctx = None
+                    db_entry = self.middleware.call_sync('user.query', [
+                        ['username', '=', user_obj['pw_name']],
+                        ['local', '=', True]
+                    ], {'select': ['sid']})
+                    if not db_entry:
+                        self.logger.error(
+                            '%s: local user exists on server but does not exist in the '
+                            'the user account table.', user_obj['pw_name']
+                        )
+                    else:
+                        sid = db_entry[0]['sid']
+                case 'ACTIVEDIRECTORY':
+                    # winbind provides idmapping for AD users
                     try:
                         idmap_ctx = idmap_winbind.WBClient()
                     except wbclient.WBCError as e:
@@ -1047,20 +1059,9 @@ def get_user_obj(self, data):
                         'id': user_obj['pw_uid']
                     })['sid']
                 except MatchNotFound:
-                    if user_obj['source'] == 'LOCAL':
-                        # Local user that doesn't have passdb entry
-                        # we can simply apply default prefix
-                        sid = SID_LOCAL_USER_PREFIX + str(user_obj['pw_uid'])
-                    else:
-                        # This is a more odd situation. The user accout exists
-                        # in IPA but doesn't have a SID assigned to it.
-                        sid = None
-            else:
-                # We were unable to establish an idmap client context even
-                # though we were able to retrieve the user account info. This
-                # most likely means that we're dealing with a local account and
-                # winbindd is not running.
-                sid = None
+                    # This is a more odd situation. Most likely case is that the user account exists
+                    # in IPA but doesn't have a SID assigned to it. All AD users have SIDs.
+                    sid = None
 
             user_obj['sid'] = sid
         else:
@@ -1832,7 +1833,8 @@ async def do_delete(self, audit_callback, pk, options):
             )
 
         group = await self.get_instance(pk)
-        audit_callback(group['name'] + (' and all its users' if options['delete_users'] else ''))
+        audit_callback(group['name'] + (' and all users that have this group as their primary group'
+                                        if options['delete_users'] else ''))
 
         if group['builtin']:
             raise CallError('A built-in group cannot be deleted.', errno.EACCES)
@@ -1932,9 +1934,24 @@ def get_group_obj(self, data):
         grp_obj['local'] = grp_obj['source'] == 'LOCAL'
 
         if data['sid_info']:
+            sid = None
+
             match grp_obj['source']:
-                case 'LOCAL' | 'ACTIVEDIRECTORY':
-                    # winbind provides idmapping for local and AD users
+                case 'LOCAL':
+                    idmap_ctx = None
+                    db_entry = self.middleware.call_sync('group.query', [
+                        ['group', '=', grp_obj['gr_name']],
+                        ['local', '=', True]
+                    ], {'select': ['sid']})
+                    if not db_entry:
+                        self.logger.error(
+                            '%s: local group exists on server but does not exist in the '
+                            'the group account table.', grp_obj['gr_name']
+                        )
+                    else:
+                        sid = db_entry[0]['sid']
+                case 'ACTIVEDIRECTORY':
+                    # winbind provides idmapping for AD groups
                     try:
                         idmap_ctx = idmap_winbind.WBClient()
                     except wbclient.WBCError as e:
@@ -1961,14 +1978,8 @@ def get_group_obj(self, data):
                         'id': grp_obj['gr_gid']
                     })['sid']
                 except MatchNotFound:
-                    if grp_obj['source'] == 'LOCAL':
-                        # Local user that doesn't have groupmap entry
-                        # we can simply apply default prefix
-                        sid = SID_LOCAL_GROUP_PREFIX + str(grp_obj['gr_gid'])
-                    else:
-                        sid = None
-            else:
-                sid = None
+                    # This can happen if IPA and group doesn't have SID assigned
+                    sid = None
 
             grp_obj['sid'] = sid
         else: