fix: preserve durable KVM runner ACL

docs/cloud-hypervisor-foundation.md

@@ -144,9 +144,13 @@ becomes live. The host bridge-forwarding
 rule is tagged with a per-run iptables comment and recorded by its exact tuple,
 so concurrent runs do not share an anonymously owned rule. Dedicated VMM
 account and device-ACL intent is persisted before `useradd`
-or `setfacl` runs. Recovery validates the account's random name, run-specific
-passwd metadata, numeric uid/gid, and exact numeric ACL entries before removing
-them.
+or `setfacl` runs. The durable record retains the first validated ACL baseline
+for every accessed device after launch-time revocation completes. Account
+deletion still journals a fresh baseline under the shared device lock, but
+restoration composes it with those durable non-VMM entries so runner access,
+mask semantics, and live peer VMM entries survive normal or stale cleanup.
+Recovery validates the account's random name, run-specific passwd metadata,
+numeric uid/gid, and exact numeric ACL entries before removing them.
 Any staged virtio-fs bind mounts are recorded by mount ID, device, root, target,
 filesystem type, and source; stale recovery revalidates and unmounts them
 deepest-first before removing their inode-validated share directory.

src/cloud-hypervisor/cleanup-handle.test.ts

@@ -42,10 +42,18 @@ describe('cleanup handle integration', () => {
         vmmIdentity: {
           accessedAclPaths?: string[];
           aclTemporaryUids?: Record<string, number[]>;
+          accessedAclBaselines?: Record<string, string[]>;
+          accessedAclTemporaryUids?: Record<string, number[]>;
         };
       };
       expect(preparedRecord.vmmIdentity.accessedAclPaths).toEqual(['/dev/kvm']);
       expect(preparedRecord.vmmIdentity.aclTemporaryUids).toEqual({ '/dev/kvm': temporaryUids });
+      expect(preparedRecord.vmmIdentity.accessedAclBaselines).toEqual({
+        '/dev/kvm': baseline,
+      });
+      expect(preparedRecord.vmmIdentity.accessedAclTemporaryUids).toEqual({
+        '/dev/kvm': temporaryUids,
+      });
 
       await handle.releaseVmmAcl('/dev/kvm');
 
@@ -55,12 +63,18 @@ describe('cleanup handle integration', () => {
           accessedAclPaths?: string[];
           aclBaselines?: object;
           aclTemporaryUids?: object;
+          accessedAclBaselines?: object;
+          accessedAclTemporaryUids?: object;
         };
       };
       expect(record.vmmIdentity.aclPaths).toEqual([]);
       expect(record.vmmIdentity.accessedAclPaths).toEqual(['/dev/kvm']);
       expect(record.vmmIdentity.aclBaselines).toEqual({});
       expect(record.vmmIdentity.aclTemporaryUids).toEqual({});
+      expect(record.vmmIdentity.accessedAclBaselines).toEqual({ '/dev/kvm': baseline });
+      expect(record.vmmIdentity.accessedAclTemporaryUids).toEqual({
+        '/dev/kvm': temporaryUids,
+      });
       await expect(handle.releaseVmmAcl('/dev/kvm')).rejects.toThrow(/intent is missing/);
     });
 

src/cloud-hypervisor/cleanup-handle.ts

@@ -118,6 +118,10 @@ export function createCleanupHandle(options: CleanupHandleFactoryOptions): Cloud
       if (!record.vmmIdentity.accessedAclPaths.includes(aclPath)) {
         record.vmmIdentity.accessedAclPaths.push(aclPath);
       }
+      record.vmmIdentity.accessedAclBaselines ??= {};
+      record.vmmIdentity.accessedAclTemporaryUids ??= {};
+      record.vmmIdentity.accessedAclBaselines[aclPath] ??= validatedBaseline;
+      record.vmmIdentity.accessedAclTemporaryUids[aclPath] ??= [...temporaryUids];
       if (!record.vmmIdentity.aclPaths.includes(aclPath)) {
         record.vmmIdentity.aclPaths.push(aclPath);
         record.vmmIdentity.aclBaselines ??= {};

src/cloud-hypervisor/cleanup-identity.test.ts

@@ -103,6 +103,47 @@ describe('cleanup identity primitives', () => {
             accessedAclPaths: ['/dev/other'],
           };
         }, /cleanup VMM identity is malformed/],
+        ['bad-accessed-acl-baseline-path', (record) => {
+          record.vmmIdentity = {
+            state: 'live',
+            name: 'awfvmm-0123456789abcdef0123',
+            uid: 23001,
+            gid: 23002,
+            aclPaths: [],
+            accessedAclPaths: ['/dev/kvm'],
+            accessedAclBaselines: {
+              '/dev/net/tun': ['user::rw-', 'group::rw-', 'other::---'],
+            },
+          };
+        }, /cleanup VMM identity is malformed/],
+        ['bad-accessed-acl-temporary-uid', (record) => {
+          record.vmmIdentity = {
+            state: 'live',
+            name: 'awfvmm-0123456789abcdef0123',
+            uid: 23001,
+            gid: 23002,
+            aclPaths: [],
+            accessedAclPaths: ['/dev/kvm'],
+            accessedAclBaselines: {
+              '/dev/kvm': [
+                'user::rw-', 'user:22001:rw-',
+                'group::rw-', 'mask::rw-', 'other::---',
+              ],
+            },
+            accessedAclTemporaryUids: { '/dev/kvm': [22002] },
+          };
+        }, /cleanup VMM identity is malformed/],
+        ['accessed-acl-temporary-list-without-baseline', (record) => {
+          record.vmmIdentity = {
+            state: 'live',
+            name: 'awfvmm-0123456789abcdef0123',
+            uid: 23001,
+            gid: 23002,
+            aclPaths: [],
+            accessedAclPaths: ['/dev/kvm'],
+            accessedAclTemporaryUids: { '/dev/kvm': [] },
+          };
+        }, /cleanup VMM identity is malformed/],
         ['pending-acl-not-accessed', (record) => {
           record.vmmIdentity = {
             state: 'live',

src/cloud-hypervisor/cleanup-identity.ts

@@ -102,6 +102,8 @@ export interface CleanupRecord {
     accessedAclPaths?: CloudHypervisorDevicePath[];
     aclBaselines?: Partial<Record<CloudHypervisorDevicePath, string[]>>;
     aclTemporaryUids?: Partial<Record<CloudHypervisorDevicePath, number[]>>;
+    accessedAclBaselines?: Partial<Record<CloudHypervisorDevicePath, string[]>>;
+    accessedAclTemporaryUids?: Partial<Record<CloudHypervisorDevicePath, number[]>>;
   };
   mounts: MountIdentity[];
   updatedAt: string;
@@ -230,6 +232,44 @@ export function validateRecord(
             identity.aclBaselines?.[aclPath as CloudHypervisorDevicePath]
               ?.includes(`user:${uid}:rw-`)))
       )) ||
+      (identity.accessedAclBaselines !== undefined && (
+        typeof identity.accessedAclBaselines !== 'object' ||
+        identity.accessedAclBaselines === null ||
+        Array.isArray(identity.accessedAclBaselines) ||
+        Object.keys(identity.accessedAclBaselines).some((aclPath) =>
+          aclPath !== '/dev/kvm' && aclPath !== '/dev/net/tun') ||
+        Object.entries(identity.accessedAclBaselines).some(([aclPath, baseline]) => {
+          try {
+            return !identity.accessedAclPaths?.includes(
+              aclPath as CloudHypervisorDevicePath,
+            ) ||
+              !Array.isArray(baseline) ||
+              parseDeviceAcl(baseline.join('\n')).length !== baseline.length;
+          } catch {
+            return true;
+          }
+        })
+      )) ||
+      (identity.accessedAclTemporaryUids !== undefined && (
+        typeof identity.accessedAclTemporaryUids !== 'object' ||
+        identity.accessedAclTemporaryUids === null ||
+        Array.isArray(identity.accessedAclTemporaryUids) ||
+        Object.keys(identity.accessedAclTemporaryUids).some((aclPath) =>
+          (aclPath !== '/dev/kvm' && aclPath !== '/dev/net/tun') ||
+          identity.accessedAclBaselines?.[
+            aclPath as CloudHypervisorDevicePath
+          ] === undefined) ||
+        Object.entries(identity.accessedAclTemporaryUids).some(([aclPath, uids]) =>
+          !identity.accessedAclPaths?.includes(
+            aclPath as CloudHypervisorDevicePath,
+          ) ||
+          !Array.isArray(uids) ||
+          uids.some((uid) => !Number.isSafeInteger(uid) || uid <= 0) ||
+          new Set(uids).size !== uids.length ||
+          !uids.every((uid) =>
+            identity.accessedAclBaselines?.[aclPath as CloudHypervisorDevicePath]
+              ?.includes(`user:${uid}:rw-`)))
+      )) ||
       (identity.state === 'live' && (() => {
         const uid = identity.uid;
         const gid = identity.gid;

src/cloud-hypervisor/cleanup-registry.test.ts

@@ -58,7 +58,7 @@ describe('DurableCloudHypervisorCleanupRegistry orchestration', () => {
       expect((await fs.stat(path.dirname(recordPath))).mode & 0o777).toBe(0o700);
     });
 
-    it('restores the exact /dev/kvm ACL baseline after run 38474851931 forced termination', async () => {
+    it('restores the durable runner ACE during stale recovery after launch intent release', async () => {
       const paths = harness.runPaths('vmm-account-recovery');
       const account = 'awfvmm-0123456789abcdef0123';
       const recordPath = path.join(
@@ -68,7 +68,15 @@ describe('DurableCloudHypervisorCleanupRegistry orchestration', () => {
       );
       let userExists = true;
       let groupExists = true;
-      let aclExists = true;
+      const runnerBaseline = [
+        'user::rw-', 'user:1001:rw-',
+        'group::rw-', 'mask::rw-', 'other::rw-',
+      ];
+      const maskOnly = ['user::rw-', 'group::rw-', 'mask::rw-', 'other::rw-'];
+      const deviceAcls = new Map([
+        ['/dev/kvm', [...maskOnly]],
+        ['/dev/net/tun', [...maskOnly]],
+      ]);
       const tools = {
         getfacl: '/usr/bin/getfacl',
         getent: '/usr/bin/getent',
@@ -96,18 +104,9 @@ describe('DurableCloudHypervisorCleanupRegistry orchestration', () => {
             : { exitCode: 2, stdout: '', stderr: '' };
         }
         if (command === tools.getfacl) {
-          if (args[args.length - 1] === '/dev/net/tun') {
-            return {
-              exitCode: 0,
-              stdout: 'user::rw-\ngroup::rw-\nmask::rw-\nother::rw-\n',
-              stderr: '',
-            };
-          }
           return {
             exitCode: 0,
-            stdout: aclExists
-              ? 'user::rw-\nuser:23001:rw-\ngroup::rw-\nmask::rw-\nother::rw-\n'
-              : 'user::rw-\ngroup::rw-\nmask::rw-\nother::rw-\n',
+            stdout: `${deviceAcls.get(args[args.length - 1])!.join('\n')}\n`,
             stderr: '',
           };
         }
@@ -116,12 +115,12 @@ describe('DurableCloudHypervisorCleanupRegistry orchestration', () => {
           if (args[0] === '-u') return { exitCode: 0, stdout: '23001\n', stderr: '' };
           return { exitCode: 0, stdout: '23002\n', stderr: '' };
         }
-        if (command === tools.setfacl) {
+        if (command === tools.setfacl && args[0].startsWith('--set=')) {
           await expect(fs.access(path.join(
             harness.temporaryRoot,
             '.device-acl-lock',
           ))).resolves.toBeUndefined();
-          aclExists = false;
+          deviceAcls.set(args[1], args[0].slice('--set='.length).split(','));
         }
         if (command === tools.userdel) {
           const persisted = JSON.parse(await fs.readFile(recordPath, 'utf8')) as {
@@ -140,18 +139,20 @@ describe('DurableCloudHypervisorCleanupRegistry orchestration', () => {
       await handle.captureArtifactSnapshot(snapshot);
       await handle.prepareVmmAccount(account);
       await handle.captureVmmIdentity({ name: account, uid: 23001, gid: 23002 });
-      const baseline = ['user::rw-', 'group::rw-', 'mask::rw-', 'other::rw-'];
-      await handle.prepareVmmAcl('/dev/kvm', baseline);
-      await handle.prepareVmmAcl('/dev/net/tun', baseline);
+      await handle.prepareVmmAcl('/dev/kvm', runnerBaseline);
+      await handle.prepareVmmAcl('/dev/net/tun', runnerBaseline);
+      await handle.releaseVmmAcl('/dev/kvm');
       await handle.releaseVmmAcl('/dev/net/tun');
       harness.state.ownerStartTime = '2000';
 
       await registry.reapPending(tools.ip, '/usr/bin/umount', tools);
 
       expect(run).toHaveBeenCalledWith(tools.setfacl, [
-        `--set=${baseline.join(',')}`,
+        `--set=${runnerBaseline.join(',')}`,
         '/dev/kvm',
       ]);
+      expect(deviceAcls.get('/dev/kvm')).toEqual(runnerBaseline);
+      expect(deviceAcls.get('/dev/net/tun')).toEqual(runnerBaseline);
       expect(run).toHaveBeenCalledWith(tools.userdel, [account]);
       expect(run).toHaveBeenCalledWith(tools.groupdel, [account]);
       await expect(fs.access(snapshot)).rejects.toMatchObject({ code: 'ENOENT' });

src/cloud-hypervisor/cleanup-vmm-identity.test.ts

@@ -34,6 +34,8 @@ function cleanupRecord(
     aclPaths?: Array<'/dev/kvm' | '/dev/net/tun'>;
     accessedAclPaths?: Array<'/dev/kvm' | '/dev/net/tun'>;
     aclTemporaryUids?: Partial<Record<'/dev/kvm' | '/dev/net/tun', number[]>>;
+    accessedAclBaselines?: Partial<Record<'/dev/kvm' | '/dev/net/tun', string[]>>;
+    accessedAclTemporaryUids?: Partial<Record<'/dev/kvm' | '/dev/net/tun', number[]>>;
   } = {},
 ): CleanupRecord {
   const runId = options.runId ?? 'acl-recovery';
@@ -70,6 +72,12 @@ function cleanupRecord(
       ...(options.aclTemporaryUids ? {
         aclTemporaryUids: options.aclTemporaryUids,
       } : {}),
+      ...(options.accessedAclBaselines ? {
+        accessedAclBaselines: options.accessedAclBaselines,
+      } : {}),
+      ...(options.accessedAclTemporaryUids ? {
+        accessedAclTemporaryUids: options.accessedAclTemporaryUids,
+      } : {}),
     },
     mounts: [],
     updatedAt: new Date(0).toISOString(),
@@ -191,15 +199,16 @@ describe('Cloud Hypervisor durable VMM identity cleanup', () => {
   });
 
   it.each([
-    ['crash after launch restoration', [], false],
-    ['partial launch-intent release', ['/dev/kvm'], false],
-    ['partial preparation retry', [], true],
+    ['cancellation recovery after launch restoration', [], false],
+    ['stale recovery after partial launch-intent release', ['/dev/kvm'], false],
+    ['partial preparation persistence retry', [], true],
   ] as const)(
-    'journals fresh baselines for every accessed device before account deletion: %s',
+    'preserves durable runner ACL history while journaling fresh deletion baselines: %s',
     async (_name, pendingPaths, retryAfterPersistFailure) => {
+      const maskOnly = ['user::rw-', 'group::rw-', 'mask::rw-', 'other::rw-'];
       const deviceAcls = new Map([
-        ['/dev/kvm', [...baseline]],
-        ['/dev/net/tun', [...baseline]],
+        ['/dev/kvm', [...maskOnly]],
+        ['/dev/net/tun', [...maskOnly]],
       ]);
       let accountExists = true;
       let groupExists = true;
@@ -209,6 +218,14 @@ describe('Cloud Hypervisor durable VMM identity cleanup', () => {
           aclPaths: [...pendingPaths],
           accessedAclPaths: ['/dev/kvm', '/dev/net/tun'],
           aclTemporaryUids: pendingPaths.length > 0 ? { '/dev/kvm': [] } : undefined,
+          accessedAclBaselines: {
+            '/dev/kvm': baseline,
+            '/dev/net/tun': baseline,
+          },
+          accessedAclTemporaryUids: {
+            '/dev/kvm': [],
+            '/dev/net/tun': [],
+          },
         },
       );
       const run = jest.fn(async (command: string, args: readonly string[]) => {
@@ -285,8 +302,8 @@ describe('Cloud Hypervisor durable VMM identity cleanup', () => {
 
       expect(record.vmmIdentity?.aclPaths).toEqual(['/dev/kvm', '/dev/net/tun']);
       expect(record.vmmIdentity?.aclBaselines).toEqual({
-        '/dev/kvm': baseline,
-        '/dev/net/tun': baseline,
+        '/dev/kvm': pendingPaths.length > 0 ? baseline : maskOnly,
+        '/dev/net/tun': maskOnly,
       });
       expect(persist).toHaveBeenCalledTimes(
         2 - pendingPaths.length + (retryAfterPersistFailure ? 1 : 0),

src/cloud-hypervisor/cleanup-vmm-identity.ts

@@ -80,12 +80,20 @@ async function prepareMissingVmmAclsLocked(
     identity.aclBaselines[aclPath] = baseline;
     identity.aclTemporaryUids ??= {};
     identity.aclTemporaryUids[aclPath] = temporaryUids;
+    identity.accessedAclBaselines ??= {};
+    identity.accessedAclTemporaryUids ??= {};
+    identity.accessedAclBaselines[aclPath] ??= baseline;
+    identity.accessedAclTemporaryUids[aclPath] ??= temporaryUids;
     try {
       if (persist) await persist();
     } catch (error) {
       identity.aclPaths.pop();
       delete identity.aclBaselines[aclPath];
       delete identity.aclTemporaryUids[aclPath];
+      if (identity.accessedAclBaselines?.[aclPath] === baseline) {
+        delete identity.accessedAclBaselines[aclPath];
+        delete identity.accessedAclTemporaryUids?.[aclPath];
+      }
       throw error;
     }
     if (baseline.some((entry) => entry.startsWith(`user:${identity.uid}:`))) {
@@ -152,6 +160,12 @@ async function restoreVmmAclsLocked(
           identity.uid!,
           identity.aclTemporaryUids?.[aclPath as CloudHypervisorDevicePath] ?? [],
           liveTemporaryUids,
+          identity.accessedAclBaselines?.[aclPath as CloudHypervisorDevicePath] ?? baseline,
+          identity.accessedAclTemporaryUids?.[
+            aclPath as CloudHypervisorDevicePath
+          ] ?? identity.aclTemporaryUids?.[
+            aclPath as CloudHypervisorDevicePath
+          ] ?? [],
         );
       }
       restoredTargets.set(aclPath as CloudHypervisorDevicePath, restoredTarget);