Fix documentation links and examples

docs/docs/api/index.md

@@ -16,7 +16,6 @@ This reference documents the Python API of Deep Lake.
 - [Query Language](query.md): Complete TQL syntax and operations
 - [Metadata](metadata.md): Dataset and column metadata management
 - [Version Control](version_control.md): Version control, history, branches, and tags
-- [Miscellaneous](misc.md): Additional auxiliary functionality
 
 ## Index Management
 

docs/docs/api/query.md

@@ -229,7 +229,7 @@ results = deeplake.query("""
         l.label
     FROM "s3://bucket1/images" AS i
     JOIN "gcs://bucket2/embeddings" AS e ON i.id = e.image_id
-    JOIN "azure://container/labels" AS l ON i.id = l.image_id
+    JOIN "azure://<account_name>/<container_name>/labels" AS l ON i.id = l.image_id
     WHERE l.confidence > 0.9
     ORDER BY COSINE_SIMILARITY(e.embedding, ARRAY[0.1, 0.2, 0.3]) DESC
     LIMIT 100

docs/docs/getting-started/quickstart.md

@@ -55,7 +55,7 @@ ds = deeplake.create("path/to/dataset")
 # Or create in cloud storage
 ds = deeplake.create("s3://my-bucket/dataset")
 ds = deeplake.create("gcs://my-bucket/dataset")
-ds = deeplake.create("azure://container/dataset")
+ds = deeplake.create("azure://<account_name>/<container_name>/<dataset_name>")
 ```
 
 ## Adding Data
@@ -179,8 +179,8 @@ cats = future.result()
 
 ## Next Steps
 
-- Explore [RAG applications](../../guide/rag)
-- Check out [Deep Learning integration](../../guide/deep-learning/deep-learning)
+- Explore [RAG applications](../guide/rag.md)
+- Check out [Deep Learning integration](../guide/deep-learning/deep-learning.ipynb)
 
 ## Support
 

docs/docs/getting-started/storage-and-creds/managed-credentials/aws/cors.md

@@ -18,7 +18,7 @@ In order to visualize Deep Lake datasets stored in your own S3 buckets in the [D
             "HEAD"
         ],
         "AllowedOrigins": [
-            "*.activeloop.ai"
+            "https://*.activeloop.ai"
         ],
         "ExposeHeaders": []
     }

docs/docs/getting-started/storage-and-creds/managed-credentials/aws/provisioning.md

@@ -129,7 +129,7 @@ The most secure method for connecting data from your AWS account to Deep Lake is
                 "kms:DescribeKey"
             ],
             "Resource": [
-                "YOUR_KMS_KEY_ARN”
+                "YOUR_KMS_KEY_ARN"
             ]
         }
     ]

docs/docs/getting-started/storage-and-creds/managed-credentials/azure/cors.md

@@ -15,11 +15,11 @@ Cross-Origin Resource Sharing (CORS) is typically enabled by default in Azure. I
 
 3\. Open the `Resource sharing (CORS)` section on the left nav.
 
-<figure><img src="images/Azure_CORS_1.png" alt=""><figcaption></figcaption></figure>
+<figure><img src="../images/Azure_CORS_1.png" alt=""><figcaption></figcaption></figure>
 
 4\. Add the following items to the permissions.
 
-<figure><img src="images/Azure_CORS_1.png" alt=""><figcaption></figcaption></figure>
+<figure><img src="../images/Azure_CORS_2.png" alt=""><figcaption></figcaption></figure>
 
 | Allowed origins           | Allowed methods | Allowed headers |
 | ------------------------- | --------------- | --------------- |

docs/docs/getting-started/storage-and-creds/managed-credentials/gcp/provisioning.md

@@ -7,7 +7,7 @@ description: "Provision Google Cloud Resources For Use With Deep Lake, Enabling
 
 ## Setting up Federated Credentials in Google Cloud Platform
 
-The most secure method for connecting data from your Azure storage to Deep Lake is using Federated Credentials, which are set up using the steps below:
+The most secure method for connecting data from your Google Cloud Storage bucket to Deep Lake is using Federated Credentials, which are set up using the steps below:
 
 #### Step 1: Create Google Cloud Service Account
 

docs/docs/guide/deep-learning/async-data-loader.md

@@ -66,11 +66,13 @@ lock = Lock()
 index = -1
 def generate_data(ds: deeplake.Dataset):
     total_count = len(ds)
+    if total_count == 0:
+        return
     global index
     while True:
         idx = 0
         with lock:
-            index = (index + 1) % (total_count - 1)
+            index = (index + 1) % total_count
             idx = index
         yield ds[idx]
 

docs/docs/guide/deep-learning/deep-learning.ipynb

@@ -175,8 +175,8 @@
    "source": [
     "# Download dataset\n",
     "from IPython.display import clear_output\n",
-    "# !wget https://github.com/activeloopai/examples/blob/main/colabs/starting_data/animals.tar\n",
-    "!curl -L -o animals.tar https://github.com/activeloopai/examples/blob/main/colabs/starting_data/animals.tar"
+    "# !wget https://raw.githubusercontent.com/activeloopai/examples/main/colabs/starting_data/animals.tar\n",
+    "!curl -L -o animals.tar https://raw.githubusercontent.com/activeloopai/examples/main/colabs/starting_data/animals.tar"
    ]
   },
   {
@@ -246,7 +246,7 @@
    "outputs": [],
    "source": [
     "# Find the class_names and list of files that need to be uploaded\n",
-    "dataset_folder = '/Users/istranic/ActiveloopCode/Datasets/animals'\n",
+    "dataset_folder = 'animals'\n",
     "\n",
     "# Find the subfolders, but filter additional files like DS_Store that are added on Mac machines.\n",
     "class_names = [item for item in os.listdir(dataset_folder) if os.path.isdir(os.path.join(dataset_folder, item))]\n",

docs/docs/guide/deep-learning/mmseg.md

@@ -111,7 +111,7 @@ data = dict(
     ),
     val=dict(
         pipeline=test_pipeline,
-        deeplake_path="hub://activeloop/coco-val-seg-mask"",
+        deeplake_path="hub://activeloop/coco-val-seg-mask",
         deeplake_tensors = {"img": "images", "gt_semantic_seg": "seg_masks"},
         deeplake_dataloader={"shuffle": False, "batch_size": 1, "num_workers": 0, "drop_last": True}
     )

docs/docs/guide/rag.md

@@ -580,7 +580,7 @@ class Document(BaseModel):
     data: Dict[str, Any]
     score: Optional[float] = None
 
-def softmax(retrieved_score: list[float], max_weight: int = 700) -> Dict[str, Document]:
+def softmax(retrieved_score: list[float], max_weight: int = 700) -> list[float]:
     # Compute the exponentials
     exp_scores = [math.exp(min(score, max_weight)) for score in retrieved_score]
 
@@ -672,7 +672,7 @@ def fusion(docs_vs: Dict[str, Document], docs_bm25: Dict[str, Document]) -> Dict
     VECTOR_WEIGHT = 0.5
     LEXICAL_WEIGHT = 0.5
 
-    results: Dict[str, Dict[str, Document]] = {}
+    results: Dict[str, Document] = {}
 
 
     for k in set(docs_vs) | set(docs_bm25):
@@ -691,16 +691,16 @@ print(results)
 
 Output:
 ```
-{'2637': Document(id='2637', data={'restaurant_name': 'Mifen101 花溪米粉王', 'restaurant_review': 'Feel like I’m back in China.'}, score=0.013747293509625419),
- '5136': Document(id='5136', data={'restaurant_name': 'Scratch', 'restaurant_review': 'Just had drinks. They were good!'}, score=0.024505473374994282),
- '17426': Document(id='17426', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks an easy going bartenders'}, score=0.024579178342433523),
- '17444': Document(id='17444', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks, good food'}, score=0.02475096426920331),
- '2496': Document(id='2496', data={'restaurant_name': 'Seasons Noodles & Dumplings Garden', 'restaurant_review': 'Comfort food, excellent service! Feel like back to home.'}, score=0.005430922978171003),
- '4022': Document(id='4022', data={'restaurant_name': 'Eureka! Mountain View', 'restaurant_review': 'Good drinks and burgers'}, score=0.0246334319067476),
- '3518': Document(id='3518', data={'restaurant_name': 'Olympus Caffe & Bakery', 'restaurant_review': 'I like the garden to sit down with friends and have a drink.'}, score=0.08915287739833623),
- '17502': Document(id='17502', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Nice place for a drink'}, score=0.02653095210662131),
- '11383': Document(id='11383', data={'restaurant_name': 'Ludwigs Biergarten Mountain View', 'restaurant_review': 'Beer is fresh tables are big feel like a proper beer garden'}, score=0.011447537567869991),
- '10788': Document(id='10788', data={'restaurant_name': 'Casa Lupe', 'restaurant_review': 'Run by a family that makes you feel like part of the family. Awesome food. I love their wet Chili Verde burritos'}, score=0.00522136854599736)}
+{'2637': Document(id='2637', data={'restaurant_name': 'Mifen101 花溪米粉王', 'restaurant_review': 'Feel like I’m back in China.'}, score=0.054989174038501676),
+ '5136': Document(id='5136', data={'restaurant_name': 'Scratch', 'restaurant_review': 'Just had drinks. They were good!'}, score=0.09802189349997713),
+ '17426': Document(id='17426', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks an easy going bartenders'}, score=0.09831671336973409),
+ '17444': Document(id='17444', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks, good food'}, score=0.09900385707681324),
+ '2496': Document(id='2496', data={'restaurant_name': 'Seasons Noodles & Dumplings Garden', 'restaurant_review': 'Comfort food, excellent service! Feel like back to home.'}, score=0.02172369191268401),
+ '4022': Document(id='4022', data={'restaurant_name': 'Eureka! Mountain View', 'restaurant_review': 'Good drinks and burgers'}, score=0.0985337276269904),
+ '3518': Document(id='3518', data={'restaurant_name': 'Olympus Caffe & Bakery', 'restaurant_review': 'I like the garden to sit down with friends and have a drink.'}, score=0.3566115095933449),
+ '17502': Document(id='17502', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Nice place for a drink'}, score=0.10612380842648524),
+ '11383': Document(id='11383', data={'restaurant_name': 'Ludwigs Biergarten Mountain View', 'restaurant_review': 'Beer is fresh tables are big feel like a proper beer garden'}, score=0.045790150271479965),
+ '10788': Document(id='10788', data={'restaurant_name': 'Casa Lupe', 'restaurant_review': 'Run by a family that makes you feel like part of the family. Awesome food. I love their wet Chili Verde burritos'}, score=0.02088547418398944)}
 ```
 
 We sort the results dictionary by each document's combined score in descending order, ensuring that the highest-ranking documents appear first.
@@ -856,6 +856,7 @@ for el, score in zip(bm25_results, bm25s):
 
 ```python
 results = fusion(docs_vs, docs_bm25)
+sorted_documents = dict(sorted(results.items(), key=lambda item: item[1].score, reverse=True))
 ```
 
 ```python
@@ -1287,8 +1288,7 @@ Dataset(columns=(text,embedding), length=19719)
 ```
 
 ```python
-medical_dataset.add_column(name="embedding", dtype=types.Array(types.Float32(),dimensions=2))
-medical_dataset.commit()
+# The copied dataset already has an embedding column, so reuse it.
 ```
 
 3. **Text Extraction** : 

docs/docs/guide/v3-conversion.md

@@ -105,6 +105,8 @@ deeplake.create = create
 
 ```python
 # 1. Create v4 dataset with desired schema
+old_ds_path = "hub://org_name/v3_dataset"  # Replace with the v3 dataset path to migrate.
+batch_size = 32
 ds = deeplake.create("s3://new/dataset")
 ds.add_column("embeddings", deeplake.types.Embedding(768))
 ds.add_column("images", deeplake.types.Image()) 
@@ -115,9 +117,10 @@ source = deeplake.query(f'SELECT * FROM "{old_ds_path}"')
 
 # 3. Migrate in batches with progress tracking
 for i in range(0, len(source), batch_size):
-    batch = source[i:i+batch_size]
+    batch = [row.to_dict() for row in source[i:i+batch_size]]
     ds.append(batch)
-    if i % 10000 == 0:
+    processed = i + len(batch)
+    if processed // 10000 > i // 10000 or processed == len(source):
         ds.commit()
 ```
 

docs/docs/llms.txt

@@ -122,7 +122,7 @@ SELECT AVG(features, axis=0) FROM dataset
 SELECT i.image, e.embedding, m.metadata
 FROM "s3://bucket1/images" AS i
 JOIN "gcs://bucket2/embeddings" AS e ON i.id = e.image_id  
-JOIN "azure://container/meta" AS m ON i.id = m.image_id
+JOIN "azure://<account_name>/<container_name>/meta" AS m ON i.id = m.image_id
 WHERE m.verified = true
 ORDER BY COSINE_SIMILARITY(e.embedding, ARRAY[0.1,0.2,0.3]) DESC
 ```