Bugfix/cstackex 327-Concurrent iSCSI VM deployment fails intermittently: post-clone LUN resize returns 404

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/SANFeignClient.java

@@ -46,7 +46,7 @@ public interface SANFeignClient {
     @Headers({"Authorization: {authHeader}"})
     OntapResponse<Lun> getLunResponse(@Param("authHeader") String authHeader, @QueryMap Map<String, Object> queryMap);
 
-    @RequestLine("GET /{uuid}")
+    @RequestLine("GET /api/storage/luns/{uuid}")
     @Headers({"Authorization: {authHeader}"})
     Lun getLunByUUID(@Param("authHeader") String authHeader, @Param("uuid") String uuid);
 

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java

@@ -231,6 +231,9 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) {
      *
      * <p>The resulting LUN shares blocks with its source and is created in constant time.
      * It is a sis-clone, so it stays readable after the source LUN is deleted.</p>
+     *
+     * <p>The create response includes the LUN uuid. That uuid is confirmed with repeated
+     * {@code GET /api/storage/luns/{uuid}} calls while ONTAP returns 404.</p>
      */
     @Override
     public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) {
@@ -249,12 +252,15 @@ public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume)
                 lunRequest.getClone().getSource().getUuid());
         try {
             String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword());
-            OntapResponse<Lun> clonedLun = sanFeignClient.createLun(authHeader, true, lunRequest);
-            if (clonedLun == null || CollectionUtils.isEmpty(clonedLun.getRecords())) {
+            OntapResponse<Lun> created = sanFeignClient.createLun(authHeader, true, lunRequest);
+            if (created == null || created.getRecords() == null || created.getRecords().isEmpty()
+                    || created.getRecords().get(0) == null
+                    || StringUtils.isBlank(created.getRecords().get(0).getUuid())) {
                 logger.error("cloneCloudStackVolume: LUN clone returned no records for Lun {}", lunRequest.getName());
                 throw new CloudRuntimeException("Failed to clone Lun: " + lunRequest.getName());
             }
-            Lun lun = clonedLun.getRecords().get(0);
+            String lunUuid = created.getRecords().get(0).getUuid();
+            Lun lun = getLunByUuidUntilVisible(authHeader, lunUuid, lunRequest.getName());
             validateCreatedLun(lun, lunRequest.getName(), "cloneCloudStackVolume");
             logger.debug("cloneCloudStackVolume: LUN cloned successfully. Lun: {}", lun);
 
@@ -274,6 +280,50 @@ public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume)
         }
     }
 
+    /**
+     * Confirms a clone uuid by reading that LUN directly.
+     *
+     * <p>{@code GET /api/storage/luns/{uuid}} returns 404 until the clone is visible. Those
+     * 404s are retried. Any other GET failure is returned to the caller.</p>
+     */
+    private Lun getLunByUuidUntilVisible(String authHeader, String lunUuid, String lunName) {
+        for (int attempt = 1; attempt <= OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES; attempt++) {
+            try {
+                Lun lun = sanFeignClient.getLunByUUID(authHeader, lunUuid);
+                if (lun != null) {
+                    if (StringUtils.isBlank(lun.getUuid())) {
+                        lun.setUuid(lunUuid);
+                    }
+                    if (lun.getName() == null) {
+                        lun.setName(lunName);
+                    }
+                    logger.debug("cloneCloudStackVolume: LUN uuid [{}] is visible on attempt {}/{}",
+                            lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+                    return lun;
+                }
+                logger.info("cloneCloudStackVolume: LUN uuid [{}] is not visible yet (attempt {}/{})",
+                        lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+            } catch (FeignException e) {
+                if (e.status() != 404) {
+                    logger.error("cloneCloudStackVolume: GET LUN uuid [{}] failed, Status: {}, Exception: {}",
+                            lunUuid, e.status(), e.getMessage());
+                    throw new CloudRuntimeException("Failed to fetch cloned Lun: " + e.getMessage());
+                }
+                logger.info("cloneCloudStackVolume: GET LUN uuid [{}] returned 404 (attempt {}/{})",
+                        lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+            }
+            if (attempt < OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES) {
+                try {
+                    Thread.sleep(OntapStorageConstants.ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS);
+                } catch (InterruptedException interrupted) {
+                    Thread.currentThread().interrupt();
+                    throw new CloudRuntimeException("Interrupted while waiting for cloned Lun " + lunName, interrupted);
+                }
+            }
+        }
+        throw new CloudRuntimeException("Cloned Lun uuid was not visible for: " + lunUuid);
+    }
+
     /**
      * Ensures ONTAP returned a usable LUN identity from create/clone. Callers in the datastore
      * driver rely on non-null name and uuid, so reject incomplete records at the Feign boundary.

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java

@@ -175,6 +175,9 @@ public class OntapStorageConstants {
     public static final int ONTAP_VOLUME_JOB_POLL_INTERVAL_MS = 1000;
     public static final int ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS = 2000;
     public static final int ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES = 30;
+    /** GET-by-UUID retries after LUN clone response. */
+    public static final int ONTAP_LUN_CLONE_GET_MAX_RETRIES = 5;
+    public static final int ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS = 1000;
 
     /** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */
     public static final String ONTAP_FLEXVOL_SNAPSHOT = "ontapFlexVolSnapshot";

plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java

@@ -1025,13 +1025,15 @@ void testCloneCloudStackVolume_Success() {
 
             when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class)))
                     .thenReturn(response);
+            when(sanFeignClient.getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"))).thenReturn(clonedLun);
 
             CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);
 
             assertNotNull(result);
             assertEquals("cloned-lun-uuid", result.getLun().getUuid());
             assertSame(volumeInfo, result.getVolumeInfo());
             verify(sanFeignClient).createLun(eq(authHeader), eq(true), any(Lun.class));
+            verify(sanFeignClient).getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"));
         }
     }
 
@@ -1074,6 +1076,7 @@ void testCloneCloudStackVolume_SnapshotSource_SendsSnapshotQualifiedName() {
             utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password"))
                     .thenReturn(authHeader);
             when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response);
+            when(sanFeignClient.getLunByUUID(eq(authHeader), eq("new-lun-uuid"))).thenReturn(createdLun);
 
             CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);